ci: untangle getting test images

This simplifies and optimizes getting container images used for tests.

Currently, we have three different ways of getting images:

1. (for hello-world) the image is in this repo under tests/integration/testdata.

2. (for busybox) download it from github (the repo that is used for
   preparing official Docker image) using curl.

3. (for debian) download from Docker hub, using skopeo and umoci.

To further complicate things, we have to do this downloading in multiple
scenarios (at least 4): locally, in github CI, from Dockefile, inside a
Vagrant VM. For each scenario, we have to install skopeo and umoci, and
those two are not yet universally available for all the distros that we
use.

Yet another complication is those images are used for tests/integration
(bats-driven tests) as well as for libcontainer/integration (go tests).
The tests in libcontainer/integration rely on busybox being available
from /busybox, and the bats tests just download the images to a
temporary location during every run.

It is also hard to support CI for other architectures, because all
the machinery for preparing images is so complicated.

This commit is an attempt to simplify and optimize getting images,
mostly by getting rid of skopeo and umoci dependencies, but also
by moving the download logic into one small shell script, which
is used from all the places.

Benefits:

 - images (if not present) are only downloaded once;
 - same images are used for both kind of tests (go and bats);
 - same images are used for local and inside-docker tests
   (because source directory is mounted into container);
 - the download logic is located within 1 simple shell script.

[v2: fix eval; more doc to get-images; print URL if curl failed]
[v3: use "slim" debian, twice as small]
[v4: fix not using $image in setup_bundle]
[v5: don't remove TESTDATA from helpers.bash]
[v6: add i386 support]

Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>

.github/workflows/test.yml

@@ -29,13 +29,10 @@ jobs:
 
     - name: install deps
       run: |
-        # skopeo repo
-        echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/ /' | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
-        wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/Debian_10/Release.key -O- | sudo apt-key add -
         # criu repo
         sudo add-apt-repository -y ppa:criu/ppa
         # apt-add-repository runs apt update so we don't have to
-        sudo apt -q install libseccomp-dev criu skopeo
+        sudo apt -q install libseccomp-dev criu
 
     - name: install go ${{ matrix.go-version }}
       uses: actions/setup-go@v2
@@ -46,26 +43,11 @@ jobs:
     - name: build
       run: sudo -E PATH="$PATH" make all
 
-    - name: install umoci
-      run: |
-        mkdir ~/bin
-        echo "PATH=$HOME/bin:$PATH" >> $GITHUB_ENV
-        curl -o ~/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/v0.4.6/umoci.amd64
-        chmod +x ~/bin/umoci
-
     - name: install bats
       uses: mig4/setup-bats@v1
       with:
         bats-version: 1.2.1
 
-    - name: get images for libcontainer/integration unit test
-      # no rootless unit tests
-      if: matrix.rootless != 'rootless'
-      run: |
-        sudo mkdir /busybox /debian
-        . tests/integration/multi-arch.bash
-        curl -fsSL `get_busybox` | sudo tar xfJC - /busybox
-
     - name: unit test
       if: matrix.rootless != 'rootless'
       run: sudo -E PATH="$PATH" -- make localunittest

Dockerfile

@@ -1,14 +1,11 @@
 ARG GO_VERSION=1.15
 ARG BATS_VERSION=v1.2.1
-ARG UMOCI_VERSION=v0.4.6
 
 FROM golang:${GO_VERSION}-buster
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN echo 'deb https://download.opensuse.org/repositories/devel:/tools:/criu/Debian_10/ /' > /etc/apt/sources.list.d/criu.list \
     && wget -nv https://download.opensuse.org/repositories/devel:/tools:/criu/Debian_10/Release.key -O- | apt-key add - \
-    && echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/ /' > /etc/apt/sources.list.d/skopeo.list \
-    && wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/Debian_10/Release.key -O- | apt-key add - \
     && dpkg --add-architecture armel \
     && dpkg --add-architecture armhf \
     && dpkg --add-architecture arm64 \
@@ -35,7 +32,6 @@ RUN echo 'deb https://download.opensuse.org/repositories/devel:/tools:/criu/Debi
         libseccomp2 \
         pkg-config \
         python-minimal \
-        skopeo \
         sudo \
         uidmap \
     && apt-get clean \
@@ -56,21 +52,4 @@ RUN cd /tmp \
     && ./install.sh /usr/local \
     && rm -rf /tmp/bats-core
 
-# install umoci
-ARG UMOCI_VERSION
-RUN curl -o /usr/local/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/${UMOCI_VERSION}/umoci.amd64 \
-    && chmod +x /usr/local/bin/umoci
-
 WORKDIR /go/src/github.com/opencontainers/runc
-
-# setup a playground for us to spawn containers in
-COPY tests/integration/multi-arch.bash tests/integration/
-ENV ROOTFS /busybox
-RUN mkdir -p "${ROOTFS}"
-RUN . tests/integration/multi-arch.bash \
-    && curl -fsSL `get_busybox` | tar xfJC - "${ROOTFS}"
-
-ENV DEBIAN_ROOTFS /debian
-RUN mkdir -p "${DEBIAN_ROOTFS}"
-RUN . tests/integration/multi-arch.bash \
-    && get_and_extract_debian "$DEBIAN_ROOTFS"

Makefile

@@ -123,8 +123,8 @@ validate:
 	$(GO) vet $(MOD_VENDOR) ./...
 
 shellcheck:
-	shellcheck tests/integration/*.bats
-	# TODO: add shellcheck for sh files
+	shellcheck tests/integration/*.bats tests/integration/*.sh
+	# TODO: add shellcheck for more sh files
 
 shfmt:
 	shfmt -ln bats -d -w tests/integration/*.bats

Vagrantfile.centos7

@@ -17,21 +17,16 @@ Vagrant.configure("2") do |config|
     # configuration
     GO_VERSION="1.15"
     BATS_VERSION="v1.2.1"
-    UMOCI_VERSION="v0.4.6"
 
     # install yum packages
     yum install -y -q epel-release
     (cd /etc/yum.repos.d && curl -O https://copr.fedorainfracloud.org/coprs/adrian/criu-el7/repo/epel-7/adrian-criu-el7-epel-7.repo)
-    yum install -y -q gcc git iptables jq glibc-static libseccomp-devel make skopeo criu
+    yum install -y -q gcc git iptables jq glibc-static libseccomp-devel make criu
     yum clean all
 
     # install Go
     curl -fsSL "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar Cxz /usr/local
 
-    # Install umoci
-    curl -o /usr/local/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/${UMOCI_VERSION}/umoci.amd64
-    chmod +x /usr/local/bin/umoci
-
     # install bats
     git clone https://github.com/bats-core/bats-core
     cd bats-core
@@ -53,10 +48,5 @@ EOF
 
     # Add a user for rootless tests
     useradd -u2000 -m -d/home/rootless -s/bin/bash rootless
-
-    # Add busybox for libcontainer/integration tests
-    . /vagrant/tests/integration/multi-arch.bash \
-        && mkdir /busybox \
-        && curl -fsSL $(get_busybox) | tar xfJC - /busybox
   SHELL
 end

Vagrantfile.fedora33

@@ -21,7 +21,7 @@ Vagrant.configure("2") do |config|
 config exclude kernel,kernel-core
 config install_weak_deps false
 update
-install iptables gcc make golang-go glibc-static libseccomp-devel bats jq git-core criu skopeo
+install iptables gcc make golang-go glibc-static libseccomp-devel bats jq git-core criu
 ts run
 EOF
     done
@@ -36,17 +36,6 @@ EOF
     cat /root/rootless.key.pub >> /home/rootless/.ssh/authorized_keys
     chown -R rootless.rootless /home/rootless
 
-    # Install umoci
-    UMOCI_VERSION=v0.4.6
-    curl -o /usr/local/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/${UMOCI_VERSION}/umoci.amd64
-    chmod +x /usr/local/bin/umoci
-
-    # Add busybox for libcontainer/integration tests
-    . /vagrant/tests/integration/multi-arch.bash \
-        && mkdir /busybox /debian \
-        && curl -fsSL $(get_busybox) | tar xfJC - /busybox \
-        && get_and_extract_debian /debian
-
     # Delegate cgroup v2 controllers to rootless user via --systemd-cgroup
     mkdir -p /etc/systemd/system/user@.service.d
     cat > /etc/systemd/system/user@.service.d/delegate.conf << EOF

libcontainer/integration/utils_test.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 	"syscall"
@@ -19,6 +20,36 @@ import (
 	"github.com/opencontainers/runc/libcontainer/configs"
 )
 
+var busyboxTar string
+
+// init makes sure the container images are downloaded,
+// and initializes busyboxTar. If images can't be downloaded,
+// we are unable to run any tests, so panic.
+func init() {
+	// Figure out path to get-images.sh. Note it won't work
+	// in case the compiled test binary is moved elsewhere.
+	_, ex, _, _ := runtime.Caller(0)
+	getImages, err := filepath.Abs(filepath.Join(filepath.Dir(ex), "..", "..", "tests", "integration", "get-images.sh"))
+	if err != nil {
+		panic(err)
+	}
+	// Call it to make sure images are downloaded, and to get the paths.
+	out, err := exec.Command(getImages).CombinedOutput()
+	if err != nil {
+		panic(fmt.Errorf("getImages error %s (output: %s)", err, out))
+	}
+	// Extract the value of BUSYBOX_IMAGE.
+	found := regexp.MustCompile(`(?m)^BUSYBOX_IMAGE=(.*)$`).FindSubmatchIndex(out)
+	if len(found) < 4 {
+		panic(fmt.Errorf("unable to find BUSYBOX_IMAGE=<value> in %q", out))
+	}
+	busyboxTar = string(out[found[2]:found[3]])
+	// Finally, check the file is present
+	if _, err := os.Stat(busyboxTar); err != nil {
+		panic(err)
+	}
+}
+
 func ptrInt(v int) *int {
 	return &v
 }
@@ -114,9 +145,9 @@ func remove(dir string) {
 // copyBusybox copies the rootfs for a busybox container created for the test image
 // into the new directory for the specific test
 func copyBusybox(dest string) error {
-	out, err := exec.Command("sh", "-c", fmt.Sprintf("cp -a /busybox/* %s/", dest)).CombinedOutput()
+	out, err := exec.Command("sh", "-c", fmt.Sprintf("tar --exclude './dev/*' -C %q -xf %q", dest, busyboxTar)).CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("copy error %q: %q", err, out)
+		return fmt.Errorf("untar error %q: %q", err, out)
 	}
 	return nil
 }

tests/integration/get-images.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# This script checks if container images needed for tests (currently
+# busybox and Debian 10 "Buster") are available locally, and downloads
+# them to testdata directory if not.
+#
+# The script is self-contained/standalone and is used from a few places
+# that need to ensure the images are downloaded. Its output is suitable
+# for consumption by shell via eval (see helpers.bash).
+#
+# XXX: Latest available images are fetched. Theoretically,
+# this can bring some instability in case of a broken image.
+# In this case, images will need to be pinned to a checksum
+# on a per-image and per-architecture basis.
+
+set -e -u -o pipefail
+
+# Root directory of integration tests.
+INTEGRATION_ROOT=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
+# Test data path.
+TESTDATA="${INTEGRATION_ROOT}/testdata"
+# Sanity check: $TESTDATA directory must exist.
+if [ ! -d "$TESTDATA" ]; then
+	echo "Bad TESTDATA directory: $TESTDATA. Aborting" >&2
+	exit 1
+fi
+
+function get() {
+	local dest="$1" url="$2"
+
+	[ -e "$dest" ] && return
+
+	# Sanity check: $TESTDATA directory must be writable.
+	if [ ! -w "$TESTDATA" ]; then
+		echo "TESTDATA directory ($TESTDATA) not writable. Aborting" >&2
+		exit 1
+	fi
+
+	if ! curl -o "$dest" -fsSL "$url"; then
+		echo "Failed to get $url" 1>&2
+		exit 1
+	fi
+}
+
+arch=$(go env GOARCH)
+# Convert from GOARCH to whatever the URLs below are using.
+case $arch in
+arm64)
+	arch=arm64v8
+	;;
+386)
+	arch=i386
+	;;
+esac
+
+# busybox
+BUSYBOX_IMAGE="$TESTDATA/busybox-${arch}.tar.xz"
+get "$BUSYBOX_IMAGE" \
+	"https://github.com/docker-library/busybox/raw/dist-${arch}/stable/glibc/busybox.tar.xz"
+echo "BUSYBOX_IMAGE=$BUSYBOX_IMAGE"
+
+# debian
+DEBIAN_IMAGE="$TESTDATA/debian-${arch}.tar.xz"
+get "$DEBIAN_IMAGE" \
+	"https://github.com/debuerreotype/docker-debian-artifacts/raw/dist-${arch}/buster/slim/rootfs.tar.xz"
+echo "DEBIAN_IMAGE=$DEBIAN_IMAGE"
+
+# hello-world is local, no need to download.
+HELLO_IMAGE="$TESTDATA/hello-world-${arch}.tar"
+echo "HELLO_IMAGE=$HELLO_IMAGE"

tests/integration/helpers.bash

@@ -3,7 +3,10 @@
 # Root directory of integration tests.
 INTEGRATION_ROOT=$(dirname "$(readlink -f "$BASH_SOURCE")")
 
-. ${INTEGRATION_ROOT}/multi-arch.bash
+# Download images, get *_IMAGE variables.
+IMAGES=$("${INTEGRATION_ROOT}"/get-images.sh)
+eval "$IMAGES"
+unset IMAGES
 
 RUNC="${INTEGRATION_ROOT}/../../runc"
 RECVTTY="${INTEGRATION_ROOT}/../../contrib/cmd/recvtty/recvtty"
@@ -11,16 +14,9 @@ RECVTTY="${INTEGRATION_ROOT}/../../contrib/cmd/recvtty/recvtty"
 # Test data path.
 TESTDATA="${INTEGRATION_ROOT}/testdata"
 
-# Busybox image
-BUSYBOX_IMAGE="$BATS_TMPDIR/busybox.tar"
+# Destinations for test containers.
 BUSYBOX_BUNDLE="$BATS_TMPDIR/busyboxtest"
-
-# hello-world in tar format
-HELLO_FILE=$(get_hello)
-HELLO_IMAGE="$TESTDATA/$HELLO_FILE"
 HELLO_BUNDLE="$BATS_TMPDIR/hello-world"
-
-# debian image
 DEBIAN_BUNDLE="$BATS_TMPDIR/debiantest"
 
 # CRIU PATH
@@ -475,48 +471,30 @@ function teardown_recvtty() {
 	rm -f "$CONSOLE_SOCKET"
 }
 
-function setup_busybox() {
+function setup_bundle() {
+	local image="$1"
+	local bundle="$2"
+
 	setup_recvtty
-	mkdir -p "$BUSYBOX_BUNDLE"/rootfs
-	if [ -e "/testdata/busybox.tar" ]; then
-		BUSYBOX_IMAGE="/testdata/busybox.tar"
-	fi
-	if [ ! -e $BUSYBOX_IMAGE ]; then
-		curl -o $BUSYBOX_IMAGE -sSL $(get_busybox)
-	fi
-	tar --exclude './dev/*' -C "$BUSYBOX_BUNDLE"/rootfs -xf "$BUSYBOX_IMAGE"
-	cd "$BUSYBOX_BUNDLE"
+	mkdir -p "$bundle"/rootfs
+	cd "$bundle"
+
+	tar --exclude './dev/*' -C rootfs -xf "$image"
+
 	runc_spec
 }
 
+function setup_busybox() {
+	setup_bundle "$BUSYBOX_IMAGE" "$BUSYBOX_BUNDLE"
+}
+
 function setup_hello() {
-	setup_recvtty
-	mkdir -p "$HELLO_BUNDLE"/rootfs
-	tar --exclude './dev/*' -C "$HELLO_BUNDLE"/rootfs -xf "$HELLO_IMAGE"
-	cd "$HELLO_BUNDLE"
-	runc_spec
+	setup_bundle "$HELLO_IMAGE" "$HELLO_BUNDLE"
 	update_config '(.. | select(.? == "sh")) |= "/hello"'
 }
 
 function setup_debian() {
-	# skopeo and umoci are not installed on the travis runner
-	if [ -n "${RUNC_USE_SYSTEMD}" ]; then
-		return
-	fi
-
-	setup_recvtty
-	mkdir -p "$DEBIAN_BUNDLE"
-
-	if [ ! -d "$DEBIAN_ROOTFS/rootfs" ]; then
-		get_and_extract_debian "$DEBIAN_BUNDLE"
-	fi
-
-	# Use the cached version
-	if [ ! -d "$DEBIAN_BUNDLE/rootfs" ]; then
-		cp -r "$DEBIAN_ROOTFS"/* "$DEBIAN_BUNDLE/"
-	fi
-
-	cd "$DEBIAN_BUNDLE"
+	setup_bundle "$DEBIAN_IMAGE" "$DEBIAN_BUNDLE"
 }
 
 function teardown_running_container() {

tests/integration/hooks.bats

@@ -48,6 +48,7 @@ function teardown() {
 		.hooks |= . + {"createRuntime": [{"path": "/bin/sh", "args": ["/bin/sh", "-c", $create_runtime_hook]}]} |
 		.hooks |= . + {"createContainer": [{"path": "/bin/sh", "args": ["/bin/sh", "-c", $create_container_hook]}]} |
 		.hooks |= . + {"startContainer": [{"path": "/bin/sh", "args": ["/bin/sh", "-c", "ldconfig"]}]} |
+		.root.readonly |= false |
 		.process.args = ["/bin/sh", "-c", "ldconfig -p | grep librunc"]' "$DEBIAN_BUNDLE"/config.json)
 	echo "${CONFIG}" >config.json
 

tests/integration/multi-arch.bash

@@ -1,44 +0,0 @@
-#!/bin/bash
-get_busybox() {
-	case $(go env GOARCH) in
-	arm64)
-		echo 'https://github.com/docker-library/busybox/raw/dist-arm64v8/stable/glibc/busybox.tar.xz'
-		;;
-	*)
-		echo 'https://github.com/docker-library/busybox/raw/dist-amd64/stable/glibc/busybox.tar.xz'
-		;;
-	esac
-}
-
-get_hello() {
-	case $(go env GOARCH) in
-	arm64)
-		echo 'hello-world-aarch64.tar'
-		;;
-	*)
-		echo 'hello-world.tar'
-		;;
-	esac
-}
-
-get_and_extract_debian() {
-	tmp=$(mktemp -d)
-	cd "$tmp"
-
-	debian="debian:3.11.6"
-
-	case $(go env GOARCH) in
-	arm64)
-		skopeo copy docker://arm64v8/debian:buster "oci:$debian"
-		;;
-	*)
-		skopeo copy docker://amd64/debian:buster "oci:$debian"
-		;;
-	esac
-
-	args="$([ -z "${ROOTLESS_TESTPATH+x}" ] && echo "--rootless")"
-	umoci unpack $args --image "$debian" "$1"
-
-	cd -
-	rm -rf "$tmp"
-}

tests/integration/testdata/.gitignore

@@ -0,0 +1,2 @@
+busybox-*.tar.xz
+debian-*.tar.xz

tests/rootless.sh

@@ -144,6 +144,10 @@ function powerset() {
 }
 features_powerset="$(powerset "${ALL_FEATURES[@]}")"
 
+# Make sure we have container images downloaded, as otherwise
+# rootless user won't be able to write to $TESTDATA.
+"$ROOT"/tests/integration/get-images.sh >/dev/null
+
 # Iterate over the powerset of all features.
 IFS=:
 for enabled_features in $features_powerset; do