diff --git a/bin/Makefile.am b/bin/Makefile.am index 684798ba0..351903517 100644 --- a/bin/Makefile.am +++ b/bin/Makefile.am @@ -25,6 +25,7 @@ dist_bin_SCRIPTS = ch-build \ ch-build2dir \ ch-builder2squash \ ch-builder2tar \ + ch-convert \ ch-dir2squash \ ch-fromhost \ ch-pull2dir \ diff --git a/bin/ch-build2dir b/bin/ch-build2dir index a4036d13a..99a9ee3f0 100755 --- a/bin/ch-build2dir +++ b/bin/ch-build2dir @@ -12,6 +12,7 @@ Usage: $ $(basename "$0") -t TAG [ARGS ...] CONTEXT OUTDIR ARGS are passed unchanged to "ch-build". +${deprecated_convert} EOF ) @@ -47,3 +48,5 @@ set -x "${ch_bin}"/ch-builder2tar "$tag" "$outdir" "${ch_bin}"/ch-tar2dir "${outdir}/${tag_fs}.tar.gz" "$outdir" rm "${outdir}/${tag_fs}.tar.gz" + +deprecated_convert_warn diff --git a/bin/ch-builder2squash b/bin/ch-builder2squash index 8badfcc1f..6c871d079 100755 --- a/bin/ch-builder2squash +++ b/bin/ch-builder2squash @@ -12,6 +12,7 @@ Usage: $ $(basename "$0") [-b BUILDER] IMAGE OUTDIR [ARGS ...] You must have sufficient privilege (via sudo) to run the Docker commands. +${deprecated_convert} EOF ) @@ -57,3 +58,5 @@ tar=${temp}/${image}.tar # Create squashfs, and clean up intermediate files and folders. "${ch_bin}/ch-dir2squash" "${temp}/${image}" "$outdir" "$@" rm -rf --one-file-system "$temp" + +deprecated_convert_warn diff --git a/bin/ch-builder2tar b/bin/ch-builder2tar index 22926907a..a0a7c4be4 100755 --- a/bin/ch-builder2tar +++ b/bin/ch-builder2tar @@ -10,6 +10,7 @@ Flatten a builder image into a Charliecloud image tarball. Usage: $ $(basename "$0") [-b BUILDER] [--nocompress] IMAGE OUTDIR +${deprecated_convert} EOF ) @@ -180,3 +181,5 @@ else rm "$tar" ls -lh "$tar_gzipped" fi + +deprecated_convert_warn diff --git a/bin/ch-convert b/bin/ch-convert new file mode 100755 index 000000000..69b41d902 --- /dev/null +++ b/bin/ch-convert @@ -0,0 +1,580 @@ +#!/bin/sh + +## preamble ################################################################## + +lib=$(cd "$(dirname "$0")" && pwd)/../lib/charliecloud +. "${lib}/base.sh" +PATH=${ch_bin}:$PATH + +# shellcheck disable=SC2034 +usage=$(cat < "$2" +} + +cv_dir_chimage () { + dir_in_validate "$1" + chimage_out_validate "$2" + INFO 'importing ...' + ch-image import "$1" "$2" # FIXME: no progress meter +} + +cv_dir_docker () { + dir_in_validate "$1" + docker_out_validate "$2" + dirtar=${tmpdir}/weirdal.tar.gz + # One could also use "docker build" with "FROM scratch" and "COPY", + # apparently saving a tar step. However, this will in fact tar the source + # directory anyway to send it to the Docker daemon. + cv_dir_tar "$1" "$dirtar" # FIXME: needlessly compresses + cv_tar_docker "$dirtar" "$2" + rm "$dirtar" +} + +cv_dir_squash () { + dir_in_validate "$1" + squash_out_validate "$2" + pflist=${tmpdir}/pseudofiles + INFO 'packing ...' + touch "$pflist" + mount_points_ensure "$1" "$pflist" + # 64kiB block size based on Shane's experiments. + # FIXME: mksquashfs(1) is incredibly noisy + mksquashfs "$1" "$2" -b 65536 -noappend -all-root -fstime 0 -pf "$pflist" + rm "$pflist" +} + +cv_dir_tar () { + dir_in_validate "$1" + tar_out_validate "$2" + # Don't add essential files & directories because that will happen later + # when converted to dir or squash. + INFO 'packing ...' + ( cd "$1" && tar czf - . ) | pv_ > "$2" +} + +cv_docker_chimage () { + docker_in_validate "$1" + chimage_out_validate "$2" + docker_out=${tmpdir}/weirdal.tar.gz + cv_docker_tar "$1" "$docker_out" # FIXME: needlessly compresses + cv_tar_chimage "$docker_out" "$2" + rm "$docker_out" +} + +cv_docker_dir () { + docker_in_validate "$1" + dir_out_validate "$2" + docker_out=${tmpdir}/weirdal.tar.gz + cv_docker_tar "$1" "$docker_out" # FIXME: needlessly compresses + cv_tar_dir "$docker_out" "$2" + rm "$docker_out" +} + +cv_docker_squash () { + docker_in_validate "$1" + squash_out_validate "$2" + docker_dir=${tmpdir}/weirdal + cv_docker_dir "$1" "$docker_dir" # FIXME: needlessly compresses + cv_dir_squash "$docker_dir" "$2" + rm -Rf --one-file-system "$docker_dir" +} + +cv_docker_tar () { + docker_in_validate "$1" + tar_out_validate "$2" + tmptar=${tmpdir}/weirdal.tar + tmpenv=${tmpdir}/weirdal.env + INFO 'exporting ...' + cid=$(docker_ create --read-only "$1" /bin/true) # cmd needed but not run + size=$(docker_ image inspect "$1" --format='{{.Size}}') + docker_ export "$cid" | pv_ -s "$size" > "$tmptar" + docker_ rm "$cid" > /dev/null + INFO 'adding environment ...' + docker_ inspect "$1" \ + --format='{{range .Config.Env}}{{println .}}{{end}}' > "$tmpenv" + tar rf "$tmptar" -b1 -P --xform="s|${tmpenv}|ch/environment|" "$tmpenv" + INFO 'compressing ...' + pv_ < "$tmptar" | gzip_ -6 > "$2" + rm "$tmptar" + rm "$tmpenv" +} + +cv_squash_chimage () { + squash_in_validate "$1" + chimage_out_validate "$2" + unsquash_dir=${tmpdir}/weirdal + cv_squash_dir "$1" "$unsquash_dir" + cv_dir_chimage "$unsquash_dir" "$2" + rm -Rf --one-file-system "$unsquash_dir" +} + +cv_squash_dir () { + squash_in_validate "$1" + dir_out_validate "$2" + # Note: unsquashfs(1) has no exclude filter, only include, so if the + # archive includes bad files like devices, this will fail. I don't know to + # what degree this will be a problem. + unsquashfs -d "$2" -user-xattrs "$1" + dir_fixup "$2" +} + +cv_squash_docker () { + squash_in_validate "$1" + docker_out_validate "$2" + unsquash_tar=${tmpdir}/weirdal.tar.gz + cv_squash_tar "$1" "$unsquash_tar" + cv_tar_docker "$unsquash_tar" "$2" + rm "$unsquash_tar" +} + +cv_squash_tar () { + squash_in_validate "$1" + tar_out_validate "$2" + unsquash_dir=${tmpdir}/weirdal + cv_squash_dir "$1" "$unsquash_dir" + cv_dir_tar "$unsquash_dir" "$2" + rm -Rf --one-file-system "$unsquash_dir" +} + +cv_tar_chimage () { + tar_in_validate "$1" + chimage_out_validate "$2" + INFO 'importing ...' + ch-image import "$1" "$2" # FIXME: no progress meter +} + +cv_tar_dir () { + tar_in_validate "$1" + # Infer decompression argument because GNU tar is unable to do so if input + # is a pipe, and we want to keep pv(1). See: + # https://www.gnu.org/software/tar/manual/tar.html#gzip + case $1 in + *.tar) + decompress= + ;; + *.tar.gz) + decompress=z + ;; + *.tar.xz) + decompress=J + ;; + *.tgz) + decompress=z + ;; + *) + echo "unknown extension: ${tarball}" 1>&2 + exit 1 + ;; + esac + dir_out_validate "$2" + INFO 'unpacking ...' + mkdir "$2" + # Use a pipe because PV ignores arguments if it's cat rather than PV. + # + # See FAQ on /dev exclusion. --no-wildcards-match-slash needed to prevent + # * matching multiple directories; tar default differs from sh behavior. + #shellcheck disable=SC2094 + pv_ -s "$(stat -c%s "$1")" < "$1" \ + | tar x$decompress -C "$2" -f - \ + --anchored --no-wildcards-match-slash \ + --exclude='dev/*' --exclude='*/dev/*' + dir_fixup "$2" +} + +cv_tar_docker () { + tar_in_validate "$1" + docker_out_validate "$2" + INFO "importing ..." + docker_ import "$1" "$2" # FIXME: no progress meter +} + +cv_tar_squash () { + tar_in_validate "$1" + squash_out_validate "$2" + tar_dir=${tmpdir}/weirdal + cv_tar_dir "$1" "$tar_dir" + cv_dir_squash "$tar_dir" "$2" + rm -Rf --one-file-system "$tar_dir" +} + + +## input/output validation functions ######################################### + +# Each of these checks whether $1 can be used as input/output descriptor for +# that format, and also whether it already exists if --no-clobber. Exit with +# error on validation failure. + +chimage_in_validate () { + img=$(chimage_path "$1") + [ -d "$img" ] || FATAL "source image not found in ch-image storage: $1" +} + +chimage_out_validate () { + img=$(chimage_path "$1") + if [ -d "$img" ] && [ -n "$no_clobber" ]; then + FATAL "exists in ch-image storage, not deleting per --no-clobber: ${1}" + fi +} + +# Validate that $1 can be used as an input directory. +dir_in_validate () { + [ -d "$1" ] || FATAL "not a directory: ${1}" +} + +dir_out_validate () { + parent_validate "$1" + # $1 must not exist, unless it looks like an image, in which case remove + # it (or error if --noclobber). + if [ -e "$1" ]; then + [ -d "$1" ] || FATAL "exists but not a directory: ${1}" + if [ -d "${1}/bin" ] && [ -d "${1}/dev" ] && [ -d "${1}/usr" ]; then + if [ -n "$no_clobber" ]; then + FATAL "exists, not deleting per --no-clobber: ${1}" + else + INFO "deleting existing image: ${1}" + rm -Rf --one-file-system "$1" + fi + else + FATAL "exists but does not appear to be an image: ${1}" + fi + fi +} + +docker_in_validate () { + digest=$(docker_ image ls -q "$1") + [ -n "$digest" ] || FATAL "source not found in Docker storage: ${1}" +} + +docker_out_validate () { + digest=$(docker_ image ls -q "$1") + if [ -n "$digest" ] && [ -n "$no_clobber" ]; then + FATAL "exists in Docker storage, not deleting per --no-clobber: ${1}" + fi +} + +squash_in_validate () { + [ -e "$1" ] || FATAL "not found: ${1}" +} + +squash_out_validate () { + parent_validate "$1" + path_noclobber "$1" +} + +tar_in_validate () { + [ -e "$1" ] || FATAL "not found: ${1}" +} + +tar_out_validate () { + case $1 in + *.tar.gz|*.tgz) + ;; + *) + FATAL "only gzipped tar output (.tar.gz or .tgz) supported" + ;; + esac + parent_validate "$1" + path_noclobber "$1" +} + + +## supporting functions ###################################################### + +# Return the path to image $1 in ch-image storage. +chimage_path () { + echo "$(ch-image storage-path)/img/$(tag_to_path "$1")" +} + +# Return basename of $2 (format $1) with no extension and filesystem-invalid +# characters removed, i.e., suitable for a new extension to be appended. Only +# extensions valid for the format $1 are considered. +desc_base () { + fmt=$1 + dsc=$2 + case $fmt in + dir) + basename "$dsc" + ;; + ch-image|docker) + tag_to_path "$dsc" + ;; + squash) + basename "$dsc" | sed -E 's/\.(sqfs|squash|squashfs|squishy)$//' + ;; + tar) + basename "$dsc" | sed -E 's/\.(t.z|tar(\.(.|..))?)$//' + ;; + *) + FATAL "invalid format: $fmt" + ;; + esac +} + +# Ensure $1 has everything needed to be an image directory. +dir_fixup () { + DEBUG "fixing up: $1" + # Make all directories writeable so we can delete later (hello, Red Hat). + find "$1" -type d -a ! -perm /200 -exec chmod u+w {} + + # If tarball had a single containing directory, move the contents up a + # level and remove the containing directory. It is non-trivial in POSIX sh + # to deal with hidden files; see https://unix.stackexchange.com/a/6397. + files=$(ls -Aq "$1") + if [ "$(echo "$files" | wc -l)" -eq 1 ]; then + ( cd "${1}/${files}" || FATAL "cd failed: ${1}/${files}" + for f in * .[!.]* ..?*; do + if [ -e "$f" ]; then mv -- "$f" ..; fi + done ) + rmdir "${1}/${files}" + fi + # Ensure mount points are present. + mount_points_ensure "$1" +} + +# Return validated format $1: if non-empty and valid, return it; if empty, +# infer format from the descriptor $2; otherwise, exit with error. +fmt_validate () { + fmt=$1 + dsc=$2 + if [ -z "$fmt" ]; then + case $dsc in + *.sqfs|*.squash|*.squashfs|*.squishy) + fmt=squash + ;; + *.tar|*.t?z|*.tar.?|*.tar.??) + fmt=tar + ;; + /*|./*) + fmt=dir + ;; + *) + if [ -n "$have_ch_image" ]; then + fmt=ch-image + elif [ -n "$have_docker" ]; then + fmt=docker + else + FATAL "descriptor looks like builder storage but no builder found: ${dsc}" + fi + ;; + esac + fi + case $fmt in + ch-image) + if [ -z "$have_ch_image" ]; then + FATAL "format ch-image invalid: ch-image not found" + fi + ;; + docker) + if [ -z "$have_docker" ]; then + FATAL "format docker invalid: docker not found" + fi + ;; + dir|squash|tar) + ;; + *) + FATAL "invalid format: ${fmt}" + ;; + esac + echo "$fmt" +} + +# Ensure mount points needed by ch-run exist in directory $1. Do nothing if +# something already exists, without dereferencing, in case it's a symlink, +# which will work for bind-mount later but won't resolve correctly now outside +# the container (e.g. linuxcontainers.org images; issue #1015). +# +# If $2 is non-empty, append missing mount points to a list of mksquashfs(1) +# "pseudo files" to that file instead of modifying $1. While pseudo files +# don't conflict with actual files, they do generate a warning. +# +# An alternative approach is to create the mount points in a temporary +# directory, then append that to the SquashFS archive. However, mksquashfs(1) +# does not merge the new files. Ff an existing file or directory is given in +# the appended directory, both go into the archive, with the second renamed +# (to "foo_1"). This make it impossible to add mount points to a directory +# that already exists; e.g., if /etc exists, /etc/resolv.conf will end up at +# /etc_1/resolv.conf. +# +# WARNING: Keep in sync with Image.unpack_init(). +mount_points_ensure () { + # directories + for i in bin dev etc mnt proc usr \ + mnt/0 mnt/1 mnt/2 mnt/3 mnt/4 mnt/5 mnt/6 mnt/7 mnt/8 mnt/9; do + if ! exist_p "${1}/${i}"; then + if [ -n "$2" ]; then + echo "${i} d 755 root root" >> "$2" + else + mkdir "${1}/${i}" + fi + fi + done + # files + for i in etc/hosts etc/resolv.conf; do + if ! exist_p "${1}/${i}"; then + if [ -n "$2" ]; then + echo "${i} f 644 root root true" >> "$2" + else + touch "${1}/${i}" + fi + fi + done +} + +# Validate the parent or enclosing directory of $1 exists. +parent_validate () { + parent=$(dirname "$1") + [ -d "$parent" ] || "not a directory: $parent" +} + +# Exit with error if $1 exists and --no-clobber was given. +path_noclobber () { + if [ -e "$1" ] && [ -n "$no_clobber" ]; then + FATAL "exists, not deleting per --no-clobber: ${1}" + fi +} + +# Set $tmpdir to be a new directory with a unique and unpredictable name, as a +# subdirectory of --tmp, $TMPDIR, or /var/tmp, whichever is first set. +tmpdir_setup () { + if [ -z "$tmpdir" ]; then + if [ -n "$TMPDIR" ]; then + tmpdir=$TMPDIR + else + tmpdir=/var/tmp + fi + fi + case $tmpdir in + /*) + ;; + *) + FATAL "temp dir must be absolute: ${tmpdir}" + ;; + esac + tmpdir=$(mktemp -d --tmpdir="$tmpdir" ch-convert.XXXXXX) +} + + +## main ###################################################################### + +while true; do + if ! parse_basic_arg "$1"; then + case $1 in + -i|--in-fmt) + shift + in_fmt=$1 + ;; + -i=*|--in-fmt=*) + in_fmt=${1#*=} + ;; + -n|--dry-run) + dry_run=yes + ;; + --no-clobber) + no_clobber=yes + ;; + -o|--out-fmt) + shift + out_fmt=$1 + ;; + -o=*|--out-fmt=*) + out_fmt=${1#*=} + ;; + --tmp) + shift + tmpdir=$1 + ;; + *) + break + ;; + esac + fi + shift +done +if [ "$#" -ne 2 ]; then + usage +fi +in_desc=$1 +out_desc=$2 +VERBOSE "verbose level: ${verbose}" + +if command -v ch-image > /dev/null 2>&1; then + have_ch_image=yes + VERBOSE 'ch-image: found' +else + VERBOSE 'ch-image: not found' +fi +if command -v docker > /dev/null 2>&1; then + have_docker=yes + VERBOSE 'docker: found' +else + VERBOSE 'docker: not found' +fi + +in_fmt=$(fmt_validate "$in_fmt" "$in_desc") +out_fmt=$(fmt_validate "$out_fmt" "$out_desc") +tmpdir_setup + +VERBOSE "temp dir: ${tmpdir}" +VERBOSE "noclobber: ${no_clobber:-will clobber}" +INFO 'input: %-8s %s' "$in_fmt" "$in_desc" +INFO 'output: %-8s %s' "$out_fmt" "$out_desc" + +if [ "$in_fmt" = "$out_fmt" ]; then + FATAL 'input and output formats must be different' +fi + +if [ -z "$dry_run" ]; then + # Dispatch to conversion function. POSIX sh does not support hyphen in + # function names, so remove it. + "cv_$(echo "$in_fmt" | tr -d '-')_$(echo "$out_fmt" | tr -d '-')" \ + "$in_desc" "$out_desc" +fi + +rmdir "$tmpdir" + +INFO 'done' diff --git a/bin/ch-dir2squash b/bin/ch-dir2squash index 711d71652..b723a7ed6 100755 --- a/bin/ch-dir2squash +++ b/bin/ch-dir2squash @@ -14,6 +14,7 @@ Usage: $ $(basename "$0") INDIR OUTDIR [ARGS ...] ARGS are passed unchanged to mksquashfs. +${deprecated_convert} EOF ) @@ -79,3 +80,5 @@ mksquashfs "$temp" "${outdir}/${image}.sqfs" -no-recovery rm -rf --one-file-system "$temp" ls -lh "${outdir}/${image}.sqfs" + +deprecated_convert_warn diff --git a/bin/ch-pull2dir b/bin/ch-pull2dir index c5168c15f..bb8a8ef83 100755 --- a/bin/ch-pull2dir +++ b/bin/ch-pull2dir @@ -15,6 +15,7 @@ Usage: $ $(basename "$0") IMAGE DEST You must have sufficient privilege (via sudo) to run Docker commands. +${deprecated_convert} EOF ) @@ -31,3 +32,5 @@ dest=$2 image=$(echo "$image" | sed 's/\//\./g') "${ch_bin}/ch-tar2dir" "${dest}/${image}.tar.gz" "$dest" rm -v "${dest}/${image}.tar.gz" + +deprecated_convert_warn diff --git a/bin/ch-pull2tar b/bin/ch-pull2tar index fafe255fa..e950061d8 100755 --- a/bin/ch-pull2tar +++ b/bin/ch-pull2tar @@ -15,6 +15,7 @@ Usage: $ $(basename "$0") IMAGE DEST You must have sufficient privilege (via sudo) to run Docker commands. +${deprecated_convert} EOF ) @@ -29,3 +30,5 @@ dest=$2 docker_ pull "$image" "${ch_bin}/ch-builder2tar" "$image" "$dest" + +deprecated_convert_warn diff --git a/bin/ch-tar2dir b/bin/ch-tar2dir index a8843fee8..8c61c40d6 100755 --- a/bin/ch-tar2dir +++ b/bin/ch-tar2dir @@ -12,6 +12,7 @@ Unpack an image tarball into a directory. Usage: $ $(basename "$0") TARBALL DIR +${deprecated_convert} EOF ) @@ -141,3 +142,5 @@ for i in etc/hosts etc/resolv.conf; do done echo "${newroot} unpacked ok" + +deprecated_convert_warn diff --git a/doc/Makefile.am b/doc/Makefile.am index a892f58dd..ee4ecec4d 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -31,6 +31,8 @@ ch-builder2tar.rst \ ch-build.rst \ ch-checkns_desc.rst \ ch-checkns.rst \ +ch-convert_desc.rst \ +ch-convert.rst \ ch-dir2squash_desc.rst \ ch-dir2squash.rst \ ch-fromhost_desc.rst \ @@ -75,6 +77,7 @@ man/ch-build2dir.1 \ man/ch-builder2squash.1 \ man/ch-builder2tar.1 \ man/ch-checkns.1 \ +man/ch-convert.1 \ man/ch-dir2squash.1 \ man/ch-fromhost.1 \ man/ch-image.1 \ diff --git a/doc/ch-build2dir_desc.rst b/doc/ch-build2dir_desc.rst index 89c801a8c..7d690108a 100644 --- a/doc/ch-build2dir_desc.rst +++ b/doc/ch-build2dir_desc.rst @@ -8,6 +8,9 @@ Synopsis Description =========== +.. warning:: This script is deprecated in favor of :code:`ch-convert`. It will + be removed in the next release. + Build a Docker image named :code:`TAG` described by a Dockerfile (default :code:`$CONTEXT/Dockerfile`) and unpack it into :code:`OUTDIR/TAG`. This is a wrapper for :code:`ch-build`, :code:`ch-builder2tar`, and :code:`ch-tar2dir`; diff --git a/doc/ch-builder2squash_desc.rst b/doc/ch-builder2squash_desc.rst index 801a8c5c3..1aa27100f 100644 --- a/doc/ch-builder2squash_desc.rst +++ b/doc/ch-builder2squash_desc.rst @@ -8,6 +8,9 @@ Synopsis Description =========== +.. warning:: This script is deprecated in favor of :code:`ch-convert`. It will + be removed in the next release. + Flattens the builder image tagged :code:`IMAGE` into a SquashFS file in :code:`OUTDIR`. diff --git a/doc/ch-builder2tar_desc.rst b/doc/ch-builder2tar_desc.rst index 6fecd326d..9b83d70ec 100644 --- a/doc/ch-builder2tar_desc.rst +++ b/doc/ch-builder2tar_desc.rst @@ -8,6 +8,9 @@ Synopsis Description =========== +.. warning:: This script is deprecated in favor of :code:`ch-convert`. It will + be removed in the next release. + Flatten the builder image tagged :code:`IMAGE` into a Charliecloud tarball in directory :code:`OUTDIR`. diff --git a/doc/ch-convert.rst b/doc/ch-convert.rst new file mode 100644 index 000000000..9f6751e1c --- /dev/null +++ b/doc/ch-convert.rst @@ -0,0 +1,8 @@ +:orphan: + +ch-convert man page ++++++++++++++++++++ + +.. include:: ./ch-convert_desc.rst +.. include:: ./bugs.rst +.. include:: ./see_also.rst diff --git a/doc/ch-convert_desc.rst b/doc/ch-convert_desc.rst new file mode 100644 index 000000000..0ede082e9 --- /dev/null +++ b/doc/ch-convert_desc.rst @@ -0,0 +1,167 @@ +Synopsis +======== + +:: + + $ ch-convert [-i FMT] [-o FMT] [OPTION ...] IN OUT + +Description +=========== + +Copy image :code:`IN` to :code:`OUT` and convert its format. Replace +:code:`OUT` if it already exists, unless :code:`--no-clobber` is specified. It +is an error if :code:`IN` and :code:`OUT` have the same format; use the +format's own tools for that case. + +:code:`ch-run` can run container images that are plain directories or +(optionally) SquashFS archives. However, images can take on a variety of other +formats as well. The main purpose of this tool is to make images in those +other formats available to :code:`ch-run`. + +For best performance, :code:`ch-convert` should be invoked only once, +producing the final format actually needed. + + :code:`IN` + Descriptor for the input image. For image builders, this is an image + reference; otherwise, it's a filesystem path. + + :code:`OUT` + Descriptor for the output image. + + :code:`-h`, :code:`--help` + Print help and exit. + + :code:`-i`, :code:`--in-fmt FMT` + Input image format is :code:`FMT`. If omitted, inferred as described below. + + :code:`-n`, :code:`--dry-run` + Don't read the input or write the output. Useful for testing format + inference. + + :code:`--no-clobber` + Error if :code:`OUT` already exists, rather than replacing it. + + :code:`-o`, :code:`--out-fmt FMT` + Output image format is :code:`FMT`; inferred if omitted. + + :code:`--tmp DIR` + A sub-directory for temporary storage is created in :code:`DIR` and + removed at the end of a successful conversion. **If this script crashes or + errors out, the temporary directory is left behind to assist in + debugging.** Storage may be needed up to twice the uncompressed size of + the image, depending on the input and output formats. Default: + :code:`$TMPDIR` if specified; otherwise :code:`/var/tmp`. + + :code:`-v`, :code:`--verbose` + Print extra chatter. Can be repeated. + +.. Notes: + + 1. It's a deliberate choice to use UNIXey options rather than the Skopeo + syntax [1], e.g. "-i docker" rather than "docker:image-name". + + [1]: https://manpages.debian.org/unstable/golang-github-containers-image/containers-transports.5.en.html + + 2. There used to be an [OUT_ARG ...] that would be passed unchanged to the + archiver, i.e. tar(1) or mksquashfs(1). However it wasn't clear there + were real use cases, and this has lots of opportunities to mess things + up. Also, it's not clear when it will be called. For example, if you + convert a directory to a tarball, then passing e.g. -J to XZ-compress + will work fine, but when converting from Docker, we just compress the + tarball we got from Docker, so in that case -J wouldn't work. + + 3. I also deliberately left out an option to change the output compression + algorithm, under the assumption that the default is good enough. This + can be revisited later IMO if needed. + + +Image formats +============= + +:code:`ch-convert` knows about these values of :code:`FMT`: + + :code:`ch-image` + Internal storage for Charliecloud's unprivileged image builder (Dockerfile + interpreter) :code:`ch-image`. + + :code:`dir` + Ordinary filesystem directory (i.e., not a mount point) containing an + unpacked image. Output directories that already exist are replaced if they + look like an image; otherwise, exit with an error. + + :code:`docker` + Internal storage for Docker. + + :code:`squash` + SquashFS filesystem archive containing the flattened image. SquashFS + archives are much like tar archives but are mountable, including by + :code:`ch-run`'s internal SquashFUSE mounting. Most systems have at least + the SquashFS-Tools installed which allows unpacking into a directory, just + like tar. Due to this greater flexibility, SquashFS is preferred to tar. + + **Note:** Conversions to and from SquashFS are quite noisy due to the + verbosity of the underlying :code:`mksquashfs(1)` and + :code:`unsquashfs(1)` tools. + + :code:`tar` + Tar archive containing the flattened image with no layer sub-archives; + i.e., the output of :code:`docker export` works but the output of + :code:`docker save` does not. Output tarballs are always gzipped and must + end in :code:`.tar.gz`; input tarballs can be any compression acceptable + to :code:`tar(1)`. + +All of these are local formats; :code:`ch-convert` does not know how to push +or pull images. + + +Format inference +================ + +:code:`ch-convert` tries to save typing by guessing formats when they are +reasonably clear. This is done against filenames, rather than file contents, +so the rules are the same for output descriptors that do not yet exist. + +Format inference is done for both :code:`IN` and :code:`OUT`. The first +matching glob below yields the inferred format. Paths need not exist in the +filesystem. + + 1. :code:`*.sqfs`, :code:`*.squash`, :code:`*.squashfs`: SquashFS. + + 2. :code:`*.tar`, :code:`*.t?z`, :code:`*.tar.?`, :code:`*.tar.??`: Tarball. + + 3. :code:`/*`, :code:`./*`, i.e. absolute path or relative path with + explicit dot: Directory. + + 4. If `ch-image` is installed: :code:`ch-image` internal storage. + + 5. If Docker is installed: Docker internal storage. + + 6. Otherwise: No format inference. + + +Examples +======== + +Typical build-to-run sequence for image :code:`foo/bar` using :code:`ch-run`'s +internal SquashFUSE code, inferring the output format:: + + $ sudo docker build -t foo/bar -f Dockerfile . + [...] + $ ch-convert foo/bar:latest /var/tmp/foobar.sqfs + input: docker foo/bar:latest + output: squashfs /var/tmp/foobar.sqfs + copying ... + done + $ ch-run /var/tmp/foobar.sqfs -- echo hello + hello + +Same conversion, but no format inference:: + + $ ch-convert -i ch-image -o squash foo/bar:latest /var/tmp/foobar.sqfs + input: docker foo/bar:latest + output: squashfs /var/tmp/foobar.sqfs + copying ... + done + + +.. LocalWords: FMT fmt diff --git a/doc/ch-dir2squash_desc.rst b/doc/ch-dir2squash_desc.rst index dd1888b2e..c2fa633f8 100644 --- a/doc/ch-dir2squash_desc.rst +++ b/doc/ch-dir2squash_desc.rst @@ -8,6 +8,9 @@ Synopsis Description =========== +.. warning:: This script is deprecated in favor of :code:`ch-convert`. It will + be removed in the next release. + Create Charliecloud SquashFS file from image directory :code:`IMGDIR` under directory :code:`OUTDIR`, named as last component of :code:`IMGDIR` plus suffix :code:`.sqfs`. diff --git a/doc/ch-image_desc.rst b/doc/ch-image_desc.rst index 6b3f24bcb..e5fb0be7a 100644 --- a/doc/ch-image_desc.rst +++ b/doc/ch-image_desc.rst @@ -155,8 +155,8 @@ it into containers by default). While you can currently poke around in the storage directory and find unpacked images runnable with :code:`ch-run`, this is not a supported use case. The -supported workflow uses :code:`ch-builder2tar` or :code:`ch-builder2squash` to -obtain a packed image; see the tutorial for details. +supported workflow uses :code:`ch-convert` to obtain a packed image; see the +tutorial for details. The storage directory format changes on no particular schedule. Often :code:`ch-image` is able to upgrade the directory; however, downgrading is not diff --git a/doc/ch-pull2dir_desc.rst b/doc/ch-pull2dir_desc.rst index 0f2299e8e..1c8750d1d 100644 --- a/doc/ch-pull2dir_desc.rst +++ b/doc/ch-pull2dir_desc.rst @@ -8,6 +8,9 @@ Synopsis Description =========== +.. warning:: This script is deprecated in favor of :code:`ch-convert`. It will + be removed in the next release. + Pull Docker image named :code:`IMAGE[:TAG]` from Docker Hub and extract it into a subdirectory of :code:`DIR`. A temporary tarball is stored in :code:`DIR`. diff --git a/doc/ch-pull2tar_desc.rst b/doc/ch-pull2tar_desc.rst index befbe640c..85282804a 100644 --- a/doc/ch-pull2tar_desc.rst +++ b/doc/ch-pull2tar_desc.rst @@ -8,6 +8,9 @@ Synopsis Description =========== +.. warning:: This script is deprecated in favor of :code:`ch-convert`. It will + be removed in the next release. + Pull a Docker image named :code:`IMAGE[:TAG]` from Docker Hub and flatten it into a Charliecloud tarball in directory :code:`OUTDIR`. diff --git a/doc/ch-run_desc.rst b/doc/ch-run_desc.rst index 298ea4a00..f4321d551 100644 --- a/doc/ch-run_desc.rst +++ b/doc/ch-run_desc.rst @@ -130,19 +130,23 @@ Image format The first is a simple directory that contains a Linux filesystem tree. This can be accomplished by: -* Charliecloud's tarball workflow: :code:`ch-builder2tar` and - :code:`ch-tar2dir` workflow. +* :code:`ch-convert` directly from :code:`ch-image` or another builder to a + directory. -* Manually mounting a filesystem archive, e.g. with :code:`ch-builder2squash` - and the external :code:`squashfuse` executable. +* Charliecloud's tarball workflow: build or pull the image, :code:`ch-convert` + it to a tarball, transfer the tarball to the target system, then + :code:`ch-convert` the tarball to a directory. + +* Manually mount a SquashFS image, e.g. with :code:`squashfuse(1)` and then + un-mount it after run with :code:`fusermount -u`. * Any other workflow that produces an appropriate directory tree. -The second is a SquashFS filesystem archive, available if :code:`ch-run` is -linked with the optional :code:`libsquashfuse_ll`. This is accomplished by -mounting the filesystem, servicing its FUSE requests, and unmounting it all -within :code:`ch-run`. See :code:`--mount` above to set the mount point -location. +The second is a SquashFS image archive mounted internally by :code:`ch-run`, +available if it's linked with the optional :code:`libsquashfuse_ll`. +:code:`ch-run` mounts the image filesystem, services all FUSE requests, and +unmounts it, all within :code:`ch-run`. See :code:`--mount` above to set the +mount point location. Prior versions of Charliecloud provided wrappers for the :code:`squashfuse` and :code:`squashfuse_ll` SquashFS mount commands and :code:`fusermount -u` diff --git a/doc/ch-tar2dir_desc.rst b/doc/ch-tar2dir_desc.rst index 0b3d44bfe..daec2afe8 100644 --- a/doc/ch-tar2dir_desc.rst +++ b/doc/ch-tar2dir_desc.rst @@ -8,6 +8,9 @@ Synopsis Description =========== +.. warning:: This script is deprecated in favor of :code:`ch-convert`. It will + be removed in the next release. + Extract the tarball :code:`TARBALL` into a subdirectory of :code:`DIR`. :code:`TARBALL` must contain a Linux filesystem image, e.g. as created by :code:`ch-builder2tar`, and be compressed with :code:`gzip` or :code:`xz`. If diff --git a/doc/charliecloud.rst b/doc/charliecloud.rst index 09a3ac351..542026fc9 100644 --- a/doc/charliecloud.rst +++ b/doc/charliecloud.rst @@ -15,6 +15,7 @@ ch-build2dir(1), ch-builder2squash(1), ch-builder2tar(1), ch-checkns(1), +ch-convert(1), ch-dir2squash(1), ch-fromhost(1), ch-image(1), diff --git a/doc/command-usage.rst b/doc/command-usage.rst index c2ebdb4ea..54c4bb6f7 100644 --- a/doc/command-usage.rst +++ b/doc/command-usage.rst @@ -26,6 +26,13 @@ Build a Charliecloud image from Dockerfile and unpack it into a directory. .. include:: ./ch-build2dir_desc.rst +ch-builder2squash ++++++++++++++++++ + +Flatten a builder image into a Charliecloud SquashFS file. + +.. include:: ./ch-builder2squash_desc.rst + ch-builder2tar ++++++++++++++ @@ -40,6 +47,13 @@ Check :code:`ch-run` prerequisites, e.g., namespaces and :code:`pivot_root(2)`. .. include:: ./ch-checkns_desc.rst +ch-convert +++++++++++ + +Convert an image from one format to another. + +.. include:: ./ch-convert_desc.rst + ch-dir2squash +++++++++++++ @@ -47,13 +61,6 @@ Create a SquashFS file from an image directory. .. include:: ./ch-dir2squash_desc.rst -ch-builder2squash -+++++++++++++++++ - -Flatten a builder image into a Charliecloud SquashFS file. - -.. include:: ./ch-builder2squash_desc.rst - ch-fromhost +++++++++++ diff --git a/doc/conf.py b/doc/conf.py index ea505387a..3f66bc672 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -260,8 +260,11 @@ ("ch-builder2tar", "ch-builder2tar", "Flatten a builder image into a Charliecloud image tarball", [], 1), - ("ch-checkns", "ch-checkns", - 'Check "ch-run" prerequisites, e.g., namespaces and "pivot_root(2)"', + ("ch-checkns", "ch-checkns", + 'Check "ch-run" prerequisites, e.g., namespaces and "pivot_root(2)"', + [], 1), + ("ch-convert", "ch-convert", + 'Convert an image from one format to another', [], 1), ("ch-dir2squash", "ch-dir2squash", "Create a SquashFS file from an image directory", diff --git a/doc/dev.rst b/doc/dev.rst index c35f3f89b..a92931669 100644 --- a/doc/dev.rst +++ b/doc/dev.rst @@ -646,16 +646,19 @@ Options: * :code:`--rpmbuild=DIR` : Use RPM build directory root :code:`DIR` (default: :code:`~/rpmbuild`). -For example, to build a version 0.9.7 RPM from the CentOS 7 image provided with -the test suite, on any system, and leave the results in :code:`~/rpmbuild/RPMS` -(note that the test suite would also build the necessary image diretory:: +For example, to build a version 0.9.7 RPM from the CentOS 7 image provided +with the test suite, on any system, and leave the results in +:code:`~/rpmbuild/RPMS` (note the test suite would also build the +necessary image directory):: - $ bin/ch-build2dir -t centos7 -f ./examples/Dockerfile.centos7 ./examples $CH_TEST_IMGDIR - $ packaging/fedora/build ${CH_TEST_IMGDIR}/centos7 0.9.7-1 + $ bin/ch-image build -t centos7 -f ./examples/Dockerfile.centos7 ./examples + $ bin/ch-convert centos7 $CH_TEST_IMGDIR/centos7 + $ packaging/fedora/build $CH_TEST_IMGDIR/centos7 0.9.7-1 To build a pre-release RPM of Git HEAD using the CentOS 7 image:: - $ bin/ch-build2dir -t centos7 -f ./examples/Dockerfile.centos7 ./examples $CH_TEST_IMGDIR + $ bin/ch-image build -t centos7 -f ./examples/Dockerfile.centos7 ./examples + $ bin/ch-convert centos7 $CH_TEST_IMGDIR/centos7 $ packaging/fedora/build ${CH_TEST_IMGDIR}/centos7 HEAD Gotchas and quirks diff --git a/doc/faq.rst b/doc/faq.rst index 02835ad8e..1bff2fa46 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -311,15 +311,18 @@ happy. .. _faq_docker2tar-size: -:code:`ch-builder2tar` gives incorrect image sizes --------------------------------------------------- +:code:`ch-convert` from Docker incorrect image sizes +---------------------------------------------------- -:code:`ch-builder2tar` often finishes before the progress bar is complete. For -example:: +When converting from Docker, :code:`ch-convert` often finishes before the +progress bar is complete. For example:: - $ ch-builder2tar mpihello /var/tmp + $ ch-convert -i docker mpihello /var/tmp/mpihello.tar.gz + input: docker mpihello + output: tar /var/tmp/mpihello.tar.gz + exporting ... 373MiB 0:00:21 [============================> ] 65% - 146M /var/tmp/mpihello.tar.gz + [...] In this case, the :code:`.tar.gz` contains 392 MB uncompressed:: @@ -334,9 +337,12 @@ But Docker thinks the image is 597 MB:: We've also seen cases where the Docker-reported size is an *under*\ estimate:: - $ ch-builder2tar spack /var/tmp + $ ch-convert -i docker spack /var/tmp/spack.tar.gz + input: docker spack + output: tar /var/tmp/spack.tar.gz + exporting ... 423MiB 0:00:22 [============================================>] 102% - 162M /var/tmp/spack.tar.gz + [...] $ zcat /var/tmp/spack.tar.gz | wc 4181186 20317858 444212736 $ sudo docker image inspect spack | fgrep -i size @@ -359,13 +365,13 @@ We cannot reliably prevent device files from being included in the tar, because often that is outside our control, e.g. :code:`docker export` produces a tarball. Thus, we must exclude them at unpacking time. -An additional complication is that :code:`ch-tar2dir` can handle tarballs both +An additional complication is that :code:`ch-convert` can read tarballs both with a single top-level directory and without, i.e. “tarbombs”. For example, best practice use of :code:`tar` on the command line produces the former, -while :code:`docker export` (perhaps via :code:`ch-builder2tar`) produces a -tarbomb. +while :code:`docker export` (invoked by :code:`ch-convert` when converting +from Docker) produces a tarbomb. -Thus, :code:`ch-tar2dir` uses :code:`tar --exclude` to exclude from unpacking +Thus, :code:`ch-convert` uses :code:`tar --exclude` to exclude from unpacking everything under :code:`./dev` and :code:`*/dev`, i.e., directory :code:`dev` appearing at either the first or second level are forced to be empty. @@ -847,4 +853,5 @@ Other approaches could be found with web searches such as "automate unattended SSH" or "SSH in cron jobs". -.. LocalWords: CAs SY Gutmann AUTH rHsFFqwwqh MrieaQ Za loc +.. LocalWords: CAs SY Gutmann AUTH rHsFFqwwqh MrieaQ Za loc mpihello +.. LocalWords: VirtualSize diff --git a/examples/distroless/Dockerfile b/examples/distroless/Dockerfile index 6cda22ac6..56e13b318 100644 --- a/examples/distroless/Dockerfile +++ b/examples/distroless/Dockerfile @@ -4,6 +4,3 @@ # The python3 image was chosen for ease of testing. FROM gcr.io/distroless/python3 COPY hello.py / - -# Without the CMD directive ch-builder2tar will fail when using docker -CMD ["/hello.py"] diff --git a/examples/mpihello/slurm.sh b/examples/mpihello/slurm.sh index 830935c58..b00c33824 100755 --- a/examples/mpihello/slurm.sh +++ b/examples/mpihello/slurm.sh @@ -27,7 +27,7 @@ module load friendly-testing module load charliecloud # Unpack image. -srun ch-tar2dir "$tar" "$imgdir" +srun ch-convert -o dir "$tar" "$imgdir" # MPI version in container. printf 'container: ' diff --git a/examples/spark/slurm.sh b/examples/spark/slurm.sh index 9d21e7620..e57cb4a8f 100755 --- a/examples/spark/slurm.sh +++ b/examples/spark/slurm.sh @@ -48,7 +48,7 @@ else fi # Unpack image -srun ch-tar2dir "$tar" "$img" +srun ch-convert -o dir "$tar" "$img" # Make Spark configuration mkdir "$conf" diff --git a/lib/base.sh b/lib/base.sh index 8d173340e..f624a9128 100644 --- a/lib/base.sh +++ b/lib/base.sh @@ -10,6 +10,40 @@ lib="${ch_bin}/../lib/charliecloud" . "${lib}/version.sh" +# Verbosity level; works the same as the Python code. +verbose=0 + +DEBUG () { + if [ "$verbose" -ge 2 ]; then + # shellcheck disable=SC2059 + printf "$@" 1>&2 + printf '\n' 1>&2 + fi +} + +FATAL () { + printf 'error: ' 1>&2 + # shellcheck disable=SC2059 + printf "$@" 1>&2 + printf '\n' 1>&2 + exit 1 +} + +INFO () { + # shellcheck disable=SC2059 + printf "$@" 1>&2 + printf '\n' 1>&2 +} + +VERBOSE () { + if [ "$verbose" -ge 1 ]; then + # shellcheck disable=SC2059 + printf "$@" 1>&2 + printf '\n' 1>&2 + fi +} + + # Don't call in a subshell or the selection will be lost. builder_choose () { if [ -z "$CH_BUILDER" ]; then @@ -31,6 +65,16 @@ builder_choose () { esac } +deprecated_convert=$(cat <&2 +} + # Return success if path $1 exists, without dereferencing links, failure # otherwise. ("test -e" dereferences.) exist_p () { @@ -69,21 +113,35 @@ pack_fmt_choose () { pack_fmt_valid "$CH_PACK_FMT" } +# Try to parse $1 as a common argument. If accepted, either exit (for things +# like --help) or return success; otherwise, return failure (i.e., not a +# common argument). +parse_basic_arg () { + case $1 in + --_lib-path) # undocumented + echo "$lib" + exit 0 + ;; + --help) + usage 0 # exits + ;; + -v|--verbose) + verbose=$((verbose+1)) + return 0 + ;; + --version) + version # exits + ;; + esac + return 1 # not a basic arg +} + parse_basic_args () { if [ "$#" -eq 0 ]; then usage 1 fi for i in "$@"; do - if [ "$i" = --_lib-path ]; then # undocumented - echo "$lib" - exit 0 - fi - if [ "$i" = --help ]; then - usage 0 - fi - if [ "$1" = --version ]; then - version - fi + parse_basic_arg "$i" || true done } @@ -92,7 +150,7 @@ parse_basic_args () { # NOTE: This is used both to name user-visible stuff like tarballs as well as # dig around in the ch-image storage directory. tag_to_path () { - echo "$1" | sed 's/\//%/g' + echo "$1" | tr '/' '%' } usage () { diff --git a/packaging/fedora/charliecloud.spec b/packaging/fedora/charliecloud.spec index 66cb20f62..0a8602f2d 100644 --- a/packaging/fedora/charliecloud.spec +++ b/packaging/fedora/charliecloud.spec @@ -132,6 +132,7 @@ ln -s "${sphinxdir}/js" %{buildroot}%{_pkgdocdir}/html/_static/js %{_bindir}/ch-builder2squash %{_bindir}/ch-builder2tar %{_bindir}/ch-checkns +%{_bindir}/ch-convert %{_bindir}/ch-dir2squash %{_bindir}/ch-fromhost %{_bindir}/ch-pull2dir @@ -145,6 +146,7 @@ ln -s "${sphinxdir}/js" %{buildroot}%{_pkgdocdir}/html/_static/js %{_mandir}/man1/ch-builder2squash.1* %{_mandir}/man1/ch-builder2tar.1* %{_mandir}/man1/ch-checkns.1* +%{_mandir}/man1/ch-convert.1* %{_mandir}/man1/ch-dir2squash.1* %{_mandir}/man1/ch-fromhost.1* %{_mandir}/man1/ch-pull2dir.1* diff --git a/test/Build.ch-build2dir b/test/Build.ch-build2dir deleted file mode 100755 index 5515205fe..000000000 --- a/test/Build.ch-build2dir +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# ch-test-scope: standard -# ch-test-builder-include: docker - -# Generate image directory using ch-build2dir and stage it for testing. - -set -ex - -srcdir=$1 -outdir=$2 -#workdir=$3 # unused - -tag=$(basename "$outdir") -outdir_parent=$(dirname "$outdir") - -cd "$srcdir" - -ch-build2dir -t "$tag" --file=./Dockerfile.build2dir . "$outdir_parent" -[[ -d $outdir ]] diff --git a/test/Build.ch-pull2dir b/test/Build.ch-pull2dir deleted file mode 100755 index 6be8bc8cb..000000000 --- a/test/Build.ch-pull2dir +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# ch-test-scope: standard -# ch-test-builder-include: docker - -# Generate image directory using ch-pull2dir and stage it for testing. - -set -e - -#srcdir=$1 # unused -tarball_gz=${2}.tar.gz -workdir=$3 - -imgtag=alpine:3.9 -tag=ch-pull2dir - -cd "$workdir" -ch-pull2dir "$imgtag" . -mv $imgtag $tag -tar czf ${tag}.tar.gz $tag -mv ${tag}.tar.gz "$tarball_gz" diff --git a/test/Build.ch-pull2tar b/test/Build.ch-pull2tar deleted file mode 100755 index 24240d5f9..000000000 --- a/test/Build.ch-pull2tar +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# ch-test-scope: standard -# ch-test-builder-include: docker - -# Generate a flattened image tarball using ch-pull2tar and stage it for -# testing. - -set -e - -#srcdir=$1 # unused -tarball_gz=${2}.tar.gz -workdir=$3 - -imgtag=alpine:3.9 - -cd "$workdir" -ch-pull2tar $imgtag . -mv ${imgtag}.tar.gz "$tarball_gz" diff --git a/test/Build.docker_pull b/test/Build.docker_pull index 1b985e7fc..9d3b1fb89 100755 --- a/test/Build.docker_pull +++ b/test/Build.docker_pull @@ -29,4 +29,4 @@ if [[ -z $hash_ ]]; then exit 1 fi -ch-builder2tar -b docker "$tag" "$(dirname "$tarball_gz")" +ch-convert -i docker "$tag" "$tarball_gz" diff --git a/test/Makefile.am b/test/Makefile.am index 55184fdf9..df1abe90f 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -37,9 +37,6 @@ sotest/sotest.c # Test files that should be executable. testfiles_exec = \ Build.centos7xz \ -Build.ch-build2dir \ -Build.ch-pull2dir \ -Build.ch-pull2tar \ Build.docker_pull \ Build.missing \ docs-sane \ diff --git a/test/make-auto.d/build_custom.bats.in b/test/make-auto.d/build_custom.bats.in index 13d4a9777..64c588a23 100644 --- a/test/make-auto.d/build_custom.bats.in +++ b/test/make-auto.d/build_custom.bats.in @@ -15,15 +15,14 @@ tarball=${tarballs[0]} if [[ $CH_PACK_FMT = squash ]]; then # Tarball provided but pack format is SquashFS; repack. - ch-tar2dir "$tarball" "$workdir" - ch-dir2squash "${workdir}/%(tag)s" "$ch_tardir" + ch-convert "$tarball" "${out}.sqfs" rm "$tarball" fi elif [[ -d $out ]]; then if [[ $CH_PACK_FMT = tar ]]; then ( cd "$out" && tar czf "${out}.tar.gz" -- * ) elif [[ $CH_PACK_FMT = squash ]]; then - ch-dir2squash "$out" "$ch_tardir" + ch-convert "$out" "${out}.sqfs" else false # unknown pack format fi diff --git a/test/make-auto.d/builder_to_archive.bats.in b/test/make-auto.d/builder_to_archive.bats.in index 9a6e260cf..41599113a 100644 --- a/test/make-auto.d/builder_to_archive.bats.in +++ b/test/make-auto.d/builder_to_archive.bats.in @@ -1,13 +1,12 @@ @test 'builder to archive %(tag)s' { scope %(scope)s if [[ $CH_PACK_FMT = squash ]]; then - archive="${ch_tardir}/%(tag)s.sqfs" - ch-builder2squash %(tag)s "$ch_tardir" + archive=${ch_tardir}/%(tag)s.sqfs else [[ $CH_PACK_FMT = tar ]] - archive="${ch_tardir}/%(tag)s.tar.gz" - ch-builder2tar %(tag)s "$ch_tardir" + archive=${ch_tardir}/%(tag)s.tar.gz fi + ch-convert -i "$CH_BUILDER" %(tag)s "$archive" archive_grep "$archive" archive_ok "$archive" } diff --git a/test/run/ch-convert.bats b/test/run/ch-convert.bats new file mode 100644 index 000000000..55e83aaf0 --- /dev/null +++ b/test/run/ch-convert.bats @@ -0,0 +1,335 @@ +load ../common + +# Testing strategy overview: +# +# The most efficient way to test conversion through all formats would be to +# start with a directory, cycle through all the formats one at a time, with +# directory being last, then compare the starting and ending directories. That +# corresponds to visiting all the cells in the matrix below, starting from one +# labeled "a", ending in one labeled "b", and skipping those labeled with a +# dash. Also, if visit n is in column i, then the next visit n+1 must be in +# row i. This approach does each conversion exactly once. +# +# output -> +# | dir | ch-image | docker | squash | tar | +# input +----------+----------+----------+----------+---------+ +# | dir | — | a | a | a | a | +# v ch-image | b | — | | | | +# docker | b | | — | | | +# squash | b | | | — | | +# tar | b | | | | — | +# +----------+----------+----------+----------+---------+ +# +# Because we start with a directory already available, this yields 5*5 - 5 - 1 +# = 19 conversions. However, I was not able to figure out a traversal order +# that would meet the constraints. +# +# Thus, we use the following algorithm. +# +# for every format i except dir: (4 iterations) +# convert start_dir -> i +# for every format j except dir: (4) +# if i≠j: convert i -> j +# convert j -> finish_dir +# compare start_dir with finish_dir +# +# This yields 4 * (3*2 + 1*1) = 28 conversions, due to excess conversions to +# dir. However, it can better isolate where the conversion went wrong, because +# the chain is 3 conversions long rather than 19. +# +# The outer loop is unrolled into four separate tests to avoid having one test +# that runs for two minutes. + + +# This is a little goofy, because several of the texts need *all* the +# builders. Thus, we (a) run only for builder ch-image but (b) +# pedantic-require Docker to also be installed. +setup () { + scope standard + [[ $CH_BUILDER = ch-image ]] || skip 'ch-image only' + if ! command -v docker > /dev/null 2>&1; then + pedantic_fail 'docker not found' + fi +} + + +# Return success if directories $1 and $2 are recursively the same, failure +# otherwise. This compares only metadata. False positives are possible if a +# file's content changes but the size and all other metadata stays the same; +# this seems unlikely. We could also use "diff -qr --no-dereference", which +# would also compare file conttent, but diff's --exclude only accepts basename +# patterns, not paths. The long list of excludes is things that don't +# round-trip through the various formats; the surprising directories (e.g. +# /dev) are because modification times seem to change. +compare () { + out=$( rsync -nv -aAX --delete "${1}/" "$2" \ + | sed -E -e '/^$/d' \ + -e '/^sending incremental file list/d' \ + -e '/^sent [0-9,]+ bytes/d' \ + -e '/^total size is/d' \ + -e '\|^deleting ch/|d' \ + -e '\|^deleting .dockerenv$|d' \ + -e '\|^deleting dev/console$|d' \ + -e '\|^deleting dev/pts/$|d' \ + -e '\|^deleting dev/shm/$|d' \ + -e '\|^./$|d' \ + -e '\|^WEIRD_AL_YANKOVIC$|d' \ + -e '\|^dev/$|d' \ + -e '\|^etc/$|d' \ + -e '\|^etc/hostname$|d' \ + -e '\|^etc/hosts$|d' \ + -e '\|^etc/resolv.conf -> /etc/resolv.conf.real$|d' \ + -e '\|^mnt/dev/dontdeleteme$|d' ) + echo "$out" + [ -z "$out" ] +} + +# Kludge to cook up the right input and output descriptors for ch-convert. +convert () { + ct=$1 + in_fmt=$2 + out_fmt=$3; + case $in_fmt in + ch-image) + in_desc=tmpimg + ;; + dir) + in_desc=$ch_timg + ;; + docker) + in_desc=tmpimg + ;; + tar) + in_desc=${BATS_TMPDIR}/convert.tar.gz + ;; + squash) + in_desc=${BATS_TMPDIR}/convert.sqfs + ;; + *) + echo "unknown input format: $in_fmt" + false + ;; + esac + case $out_fmt in + ch-image) + out_desc=tmpimg + ;; + dir) + out_desc=${BATS_TMPDIR}/convert.dir + ;; + docker) + out_desc=tmpimg + ;; + tar) + out_desc=${BATS_TMPDIR}/convert.tar.gz + ;; + squash) + out_desc=${BATS_TMPDIR}/convert.sqfs + ;; + *) + echo "unknown output format: $out_fmt" + false + ;; + esac + echo "CONVERT ${ct}: ${in_desc} ($in_fmt) -> ${out_desc} (${out_fmt})" + delete "$out_fmt" "$out_desc" + ch-convert --no-clobber -v -i "$in_fmt" -o "$out_fmt" "$in_desc" "$out_desc" + # Doing it twice doubles the time but also tests that both new conversions + # and overwrite work. Hence, full scope only. + if [[ $CH_TEST_SCOPE = full ]]; then + ch-convert -v -i "$in_fmt" -o "$out_fmt" "$in_desc" "$out_desc" + fi +} + +delete () { + fmt=$1 + desc=$2 + case $fmt in + ch-image) + ch-image delete "$desc" || true + ;; + dir) + rm -Rf --one-file-system "$desc" + ;; + docker) + docker_ rmi -f "$desc" + ;; + tar) + rm -f "$desc" + ;; + squash) + rm -f "$desc" + ;; + *) + echo "unknown format: $fmt" + false + ;; + esac +} + +# Test conversions dir -> $1 -> (all) -> dir. +test_from () { + end=${BATS_TMPDIR}/convert.dir + ct=1 + convert "$ct" dir "$1" + for j in ch-image docker squash tar; do + if [[ $1 != "$j" ]]; then + ct=$((ct+1)) + convert "$ct" "$1" "$j" + fi + ct=$((ct+1)) + convert "$ct" "$1" dir + image_ok "$end" + compare "$ch_timg" "$end" + done +} + + +@test 'ch-convert: format inference' { + # Test input only; output uses same code. Test cases match all the + # criteria to validate the priority. We don't exercise every possible + # descriptor pattern, only those I thought had potential for error. + + # SquashFS + run ch-convert -n ./foo:bar.sqfs out.tar + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'input: squash'* ]] + + # tar + run ch-convert -n ./foo:bar.tar out.sqfs + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'input: tar'* ]] + run ch-convert -n ./foo:bar.tgz out.sqfs + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'input: tar'* ]] + run ch-convert -n ./foo:bar.tar.Z out.sqfs + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'input: tar'* ]] + run ch-convert -n ./foo:bar.tar.gz out.sqfs + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'input: tar'* ]] + + # directory + run ch-convert -n ./foo:bar out.tar + echo "$output" + [[ $status -eq 0 ]] + [[ $output = *'input: dir'* ]] + + # builders + run ch-convert -n foo out.tar + echo "$output" + if command -v ch-image > /dev/null 2>&1; then + [[ $status -eq 0 ]] + [[ $output = *'input: ch-image'* ]] + elif command -v docker > /dev/null 2>&1; then + [[ $status -eq 0 ]] + [[ $output = *'input: docker'* ]] + else + [[ $status -eq 1 ]] + [[ $output = *'no builder found' ]] + fi +} + + +@test 'ch-convert: errors' { + # same format + run ch-convert -n foo.tar foo.tar.gz + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *'error: input and output formats must be different'* ]] + + # output directory not an image + touch "${BATS_TMPDIR}/foo.tar" + run ch-convert "${BATS_TMPDIR}/foo.tar" "$BATS_TMPDIR" + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"error: exists but does not appear to be an image: ${BATS_TMPDIR}"* ]] + rm "${BATS_TMPDIR}/foo.tar" +} + + +@test 'ch-convert: --no-clobber' { + # ch-image + printf 'FROM alpine:3.9\n' | ch-image build -t tmpimg -f - "$BATS_TMPDIR" + run ch-convert --no-clobber -o ch-image "$BATS_TMPDIR" tmpimg + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"error: exists in ch-image storage, not deleting per --no-clobber: tmpimg" ]] + + # dir + ch-convert -i ch-image -o dir 00_tiny "$BATS_TMPDIR"/00_tiny + run ch-convert --no-clobber -i ch-image -o dir 00_tiny "$BATS_TMPDIR"/00_tiny + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"error: exists, not deleting per --no-clobber: ${BATS_TMPDIR}/00_tiny" ]] + rm -Rf --one-file-system "$BATS_TMPDIR"/00_tiny + + # docker + printf 'FROM alpine:3.9\n' | docker_ build -t tmpimg - + run ch-convert --no-clobber -o docker "$BATS_TMPDIR" tmpimg + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"error: exists in Docker storage, not deleting per --no-clobber: tmpimg" ]] + + # squash + touch "${BATS_TMPDIR}/00_tiny.sqfs" + run ch-convert --no-clobber -i ch-image -o squash 00_tiny "$BATS_TMPDIR"/00_tiny.sqfs + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"error: exists, not deleting per --no-clobber: ${BATS_TMPDIR}/00_tiny.sqfs" ]] + rm "${BATS_TMPDIR}/00_tiny.sqfs" + + # tar + touch "${BATS_TMPDIR}/00_tiny.tar.gz" + run ch-convert --no-clobber -i ch-image -o tar 00_tiny "$BATS_TMPDIR"/00_tiny.tar.gz + echo "$output" + [[ $status -eq 1 ]] + [[ $output = *"error: exists, not deleting per --no-clobber: ${BATS_TMPDIR}/00_tiny.tar.gz" ]] + rm "${BATS_TMPDIR}/00_tiny.tar.gz" +} + + +@test 'ch-convert: pathological tarballs' { + [[ $CH_PACK_FMT = tar ]] || skip 'tar mode only' + out=${BATS_TMPDIR}/convert.dir + # Are /dev fixtures present in tarball? (issue #157) + present=$(tar tf "$ch_ttar" | grep -F deleteme) + [[ $(echo "$present" | wc -l) -eq 4 ]] + echo "$present" | grep -E '^img/dev/deleteme$' + echo "$present" | grep -E '^./dev/deleteme$' + echo "$present" | grep -E '^dev/deleteme$' + echo "$present" | grep -E '^img/mnt/dev/dontdeleteme$' + # Convert to dir. + ch-convert "$ch_ttar" "$out" + image_ok "$out" + # Did we raise hidden files correctly? + [[ -e ${out}/.hiddenfile1 ]] + [[ -e ${out}/..hiddenfile2 ]] + [[ -e ${out}/...hiddenfile3 ]] + # Did we remove the right /dev stuff? + [[ -e ${out}/mnt/dev/dontdeleteme ]] + [[ $(ls -Aq "${out}/dev") -eq 0 ]] + ch-run "$out" -- test -e /mnt/dev/dontdeleteme +} + + +@test 'ch-convert: dir -> ch-image -> X' { + test_from ch-image +} + +@test 'ch-convert: dir -> docker -> X' { + test_from docker +} + +@test 'ch-convert: dir -> squash -> X' { + test_from squash +} + +@test 'ch-convert: dir -> tar -> X' { + test_from tar +} diff --git a/test/run/ch-run_misc.bats b/test/run/ch-run_misc.bats index e586fd2f7..19dd158f2 100644 --- a/test/run/ch-run_misc.bats +++ b/test/run/ch-run_misc.bats @@ -18,8 +18,7 @@ load ../common scope quick run ch-run "$ch_timg" sh <