Merge pull request #2741 from kolyshkin/test-images

This commit is contained in:
Akihiro Suda
2021-02-05 15:30:43 +09:00
committed by GitHub
14 changed files with 157 additions and 169 deletions
+1 -19
View File
@@ -29,13 +29,10 @@ jobs:
- name: install deps
run: |
# skopeo repo
echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/ /' | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/Debian_10/Release.key -O- | sudo apt-key add -
# criu repo
sudo add-apt-repository -y ppa:criu/ppa
# apt-add-repository runs apt update so we don't have to
sudo apt -q install libseccomp-dev criu skopeo
sudo apt -q install libseccomp-dev criu
- name: install go ${{ matrix.go-version }}
uses: actions/setup-go@v2
@@ -46,26 +43,11 @@ jobs:
- name: build
run: sudo -E PATH="$PATH" make all
- name: install umoci
run: |
mkdir ~/bin
echo "PATH=$HOME/bin:$PATH" >> $GITHUB_ENV
curl -o ~/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/v0.4.6/umoci.amd64
chmod +x ~/bin/umoci
- name: install bats
uses: mig4/setup-bats@v1
with:
bats-version: 1.2.1
- name: get images for libcontainer/integration unit test
# no rootless unit tests
if: matrix.rootless != 'rootless'
run: |
sudo mkdir /busybox /debian
. tests/integration/multi-arch.bash
curl -fsSL `get_busybox` | sudo tar xfJC - /busybox
- name: unit test
if: matrix.rootless != 'rootless'
run: sudo -E PATH="$PATH" -- make localunittest
-21
View File
@@ -1,14 +1,11 @@
ARG GO_VERSION=1.15
ARG BATS_VERSION=v1.2.1
ARG UMOCI_VERSION=v0.4.6
FROM golang:${GO_VERSION}-buster
ARG DEBIAN_FRONTEND=noninteractive
RUN echo 'deb https://download.opensuse.org/repositories/devel:/tools:/criu/Debian_10/ /' > /etc/apt/sources.list.d/criu.list \
&& wget -nv https://download.opensuse.org/repositories/devel:/tools:/criu/Debian_10/Release.key -O- | apt-key add - \
&& echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/ /' > /etc/apt/sources.list.d/skopeo.list \
&& wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/Debian_10/Release.key -O- | apt-key add - \
&& dpkg --add-architecture armel \
&& dpkg --add-architecture armhf \
&& dpkg --add-architecture arm64 \
@@ -35,7 +32,6 @@ RUN echo 'deb https://download.opensuse.org/repositories/devel:/tools:/criu/Debi
libseccomp2 \
pkg-config \
python-minimal \
skopeo \
sudo \
uidmap \
&& apt-get clean \
@@ -56,21 +52,4 @@ RUN cd /tmp \
&& ./install.sh /usr/local \
&& rm -rf /tmp/bats-core
# install umoci
ARG UMOCI_VERSION
RUN curl -o /usr/local/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/${UMOCI_VERSION}/umoci.amd64 \
&& chmod +x /usr/local/bin/umoci
WORKDIR /go/src/github.com/opencontainers/runc
# setup a playground for us to spawn containers in
COPY tests/integration/multi-arch.bash tests/integration/
ENV ROOTFS /busybox
RUN mkdir -p "${ROOTFS}"
RUN . tests/integration/multi-arch.bash \
&& curl -fsSL `get_busybox` | tar xfJC - "${ROOTFS}"
ENV DEBIAN_ROOTFS /debian
RUN mkdir -p "${DEBIAN_ROOTFS}"
RUN . tests/integration/multi-arch.bash \
&& get_and_extract_debian "$DEBIAN_ROOTFS"
+2 -2
View File
@@ -123,8 +123,8 @@ validate:
$(GO) vet $(MOD_VENDOR) ./...
shellcheck:
shellcheck tests/integration/*.bats
# TODO: add shellcheck for sh files
shellcheck tests/integration/*.bats tests/integration/*.sh tests/*.sh
# TODO: add shellcheck for more sh files
shfmt:
shfmt -ln bats -d -w tests/integration/*.bats
+1 -11
View File
@@ -17,21 +17,16 @@ Vagrant.configure("2") do |config|
# configuration
GO_VERSION="1.15"
BATS_VERSION="v1.2.1"
UMOCI_VERSION="v0.4.6"
# install yum packages
yum install -y -q epel-release
(cd /etc/yum.repos.d && curl -O https://copr.fedorainfracloud.org/coprs/adrian/criu-el7/repo/epel-7/adrian-criu-el7-epel-7.repo)
yum install -y -q gcc git iptables jq glibc-static libseccomp-devel make skopeo criu
yum install -y -q gcc git iptables jq glibc-static libseccomp-devel make criu
yum clean all
# install Go
curl -fsSL "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar Cxz /usr/local
# Install umoci
curl -o /usr/local/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/${UMOCI_VERSION}/umoci.amd64
chmod +x /usr/local/bin/umoci
# install bats
git clone https://github.com/bats-core/bats-core
cd bats-core
@@ -53,10 +48,5 @@ EOF
# Add a user for rootless tests
useradd -u2000 -m -d/home/rootless -s/bin/bash rootless
# Add busybox for libcontainer/integration tests
. /vagrant/tests/integration/multi-arch.bash \
&& mkdir /busybox \
&& curl -fsSL $(get_busybox) | tar xfJC - /busybox
SHELL
end
+1 -12
View File
@@ -21,7 +21,7 @@ Vagrant.configure("2") do |config|
config exclude kernel,kernel-core
config install_weak_deps false
update
install iptables gcc make golang-go glibc-static libseccomp-devel bats jq git-core criu skopeo
install iptables gcc make golang-go glibc-static libseccomp-devel bats jq git-core criu
ts run
EOF
done
@@ -36,17 +36,6 @@ EOF
cat /root/rootless.key.pub >> /home/rootless/.ssh/authorized_keys
chown -R rootless.rootless /home/rootless
# Install umoci
UMOCI_VERSION=v0.4.6
curl -o /usr/local/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/${UMOCI_VERSION}/umoci.amd64
chmod +x /usr/local/bin/umoci
# Add busybox for libcontainer/integration tests
. /vagrant/tests/integration/multi-arch.bash \
&& mkdir /busybox /debian \
&& curl -fsSL $(get_busybox) | tar xfJC - /busybox \
&& get_and_extract_debian /debian
# Delegate cgroup v2 controllers to rootless user via --systemd-cgroup
mkdir -p /etc/systemd/system/user@.service.d
cat > /etc/systemd/system/user@.service.d/delegate.conf << EOF
+33 -2
View File
@@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"syscall"
@@ -19,6 +20,36 @@ import (
"github.com/opencontainers/runc/libcontainer/configs"
)
var busyboxTar string
// init makes sure the container images are downloaded,
// and initializes busyboxTar. If images can't be downloaded,
// we are unable to run any tests, so panic.
func init() {
// Figure out path to get-images.sh. Note it won't work
// in case the compiled test binary is moved elsewhere.
_, ex, _, _ := runtime.Caller(0)
getImages, err := filepath.Abs(filepath.Join(filepath.Dir(ex), "..", "..", "tests", "integration", "get-images.sh"))
if err != nil {
panic(err)
}
// Call it to make sure images are downloaded, and to get the paths.
out, err := exec.Command(getImages).CombinedOutput()
if err != nil {
panic(fmt.Errorf("getImages error %s (output: %s)", err, out))
}
// Extract the value of BUSYBOX_IMAGE.
found := regexp.MustCompile(`(?m)^BUSYBOX_IMAGE=(.*)$`).FindSubmatchIndex(out)
if len(found) < 4 {
panic(fmt.Errorf("unable to find BUSYBOX_IMAGE=<value> in %q", out))
}
busyboxTar = string(out[found[2]:found[3]])
// Finally, check the file is present
if _, err := os.Stat(busyboxTar); err != nil {
panic(err)
}
}
func ptrInt(v int) *int {
return &v
}
@@ -114,9 +145,9 @@ func remove(dir string) {
// copyBusybox copies the rootfs for a busybox container created for the test image
// into the new directory for the specific test
func copyBusybox(dest string) error {
out, err := exec.Command("sh", "-c", fmt.Sprintf("cp -a /busybox/* %s/", dest)).CombinedOutput()
out, err := exec.Command("sh", "-c", fmt.Sprintf("tar --exclude './dev/*' -C %q -xf %q", dest, busyboxTar)).CombinedOutput()
if err != nil {
return fmt.Errorf("copy error %q: %q", err, out)
return fmt.Errorf("untar error %q: %q", err, out)
}
return nil
}
+70
View File
@@ -0,0 +1,70 @@
#!/bin/bash
# This script checks if container images needed for tests (currently
# busybox and Debian 10 "Buster") are available locally, and downloads
# them to testdata directory if not.
#
# The script is self-contained/standalone and is used from a few places
# that need to ensure the images are downloaded. Its output is suitable
# for consumption by shell via eval (see helpers.bash).
#
# XXX: Latest available images are fetched. Theoretically,
# this can bring some instability in case of a broken image.
# In this case, images will need to be pinned to a checksum
# on a per-image and per-architecture basis.
set -e -u -o pipefail
# Root directory of integration tests.
INTEGRATION_ROOT=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
# Test data path.
TESTDATA="${INTEGRATION_ROOT}/testdata"
# Sanity check: $TESTDATA directory must exist.
if [ ! -d "$TESTDATA" ]; then
echo "Bad TESTDATA directory: $TESTDATA. Aborting" >&2
exit 1
fi
function get() {
local dest="$1" url="$2"
[ -e "$dest" ] && return
# Sanity check: $TESTDATA directory must be writable.
if [ ! -w "$TESTDATA" ]; then
echo "TESTDATA directory ($TESTDATA) not writable. Aborting" >&2
exit 1
fi
if ! curl -o "$dest" -fsSL "$url"; then
echo "Failed to get $url" 1>&2
exit 1
fi
}
arch=$(go env GOARCH)
# Convert from GOARCH to whatever the URLs below are using.
case $arch in
arm64)
arch=arm64v8
;;
386)
arch=i386
;;
esac
# busybox
BUSYBOX_IMAGE="$TESTDATA/busybox-${arch}.tar.xz"
get "$BUSYBOX_IMAGE" \
"https://github.com/docker-library/busybox/raw/dist-${arch}/stable/glibc/busybox.tar.xz"
echo "BUSYBOX_IMAGE=$BUSYBOX_IMAGE"
# debian
DEBIAN_IMAGE="$TESTDATA/debian-${arch}.tar.xz"
get "$DEBIAN_IMAGE" \
"https://github.com/debuerreotype/docker-debian-artifacts/raw/dist-${arch}/buster/slim/rootfs.tar.xz"
echo "DEBIAN_IMAGE=$DEBIAN_IMAGE"
# hello-world is local, no need to download.
HELLO_IMAGE="$TESTDATA/hello-world-${arch}.tar"
echo "HELLO_IMAGE=$HELLO_IMAGE"
+20 -43
View File
@@ -3,25 +3,20 @@
# Root directory of integration tests.
INTEGRATION_ROOT=$(dirname "$(readlink -f "$BASH_SOURCE")")
. ${INTEGRATION_ROOT}/multi-arch.bash
# Download images, get *_IMAGE variables.
IMAGES=$("${INTEGRATION_ROOT}"/get-images.sh)
eval "$IMAGES"
unset IMAGES
RUNC="${INTEGRATION_ROOT}/../../runc"
RECVTTY="${INTEGRATION_ROOT}/../../contrib/cmd/recvtty/recvtty"
GOPATH="$(mktemp -d --tmpdir runc-integration-gopath.XXXXXX)"
# Test data path.
TESTDATA="${INTEGRATION_ROOT}/testdata"
# Busybox image
BUSYBOX_IMAGE="$BATS_TMPDIR/busybox.tar"
# Destinations for test containers.
BUSYBOX_BUNDLE="$BATS_TMPDIR/busyboxtest"
# hello-world in tar format
HELLO_FILE=$(get_hello)
HELLO_IMAGE="$TESTDATA/$HELLO_FILE"
HELLO_BUNDLE="$BATS_TMPDIR/hello-world"
# debian image
DEBIAN_BUNDLE="$BATS_TMPDIR/debiantest"
# CRIU PATH
@@ -476,48 +471,30 @@ function teardown_recvtty() {
rm -f "$CONSOLE_SOCKET"
}
function setup_busybox() {
function setup_bundle() {
local image="$1"
local bundle="$2"
setup_recvtty
mkdir -p "$BUSYBOX_BUNDLE"/rootfs
if [ -e "/testdata/busybox.tar" ]; then
BUSYBOX_IMAGE="/testdata/busybox.tar"
fi
if [ ! -e $BUSYBOX_IMAGE ]; then
curl -o $BUSYBOX_IMAGE -sSL $(get_busybox)
fi
tar --exclude './dev/*' -C "$BUSYBOX_BUNDLE"/rootfs -xf "$BUSYBOX_IMAGE"
cd "$BUSYBOX_BUNDLE"
mkdir -p "$bundle"/rootfs
cd "$bundle"
tar --exclude './dev/*' -C rootfs -xf "$image"
runc_spec
}
function setup_busybox() {
setup_bundle "$BUSYBOX_IMAGE" "$BUSYBOX_BUNDLE"
}
function setup_hello() {
setup_recvtty
mkdir -p "$HELLO_BUNDLE"/rootfs
tar --exclude './dev/*' -C "$HELLO_BUNDLE"/rootfs -xf "$HELLO_IMAGE"
cd "$HELLO_BUNDLE"
runc_spec
setup_bundle "$HELLO_IMAGE" "$HELLO_BUNDLE"
update_config '(.. | select(.? == "sh")) |= "/hello"'
}
function setup_debian() {
# skopeo and umoci are not installed on the travis runner
if [ -n "${RUNC_USE_SYSTEMD}" ]; then
return
fi
setup_recvtty
mkdir -p "$DEBIAN_BUNDLE"
if [ ! -d "$DEBIAN_ROOTFS/rootfs" ]; then
get_and_extract_debian "$DEBIAN_BUNDLE"
fi
# Use the cached version
if [ ! -d "$DEBIAN_BUNDLE/rootfs" ]; then
cp -r "$DEBIAN_ROOTFS"/* "$DEBIAN_BUNDLE/"
fi
cd "$DEBIAN_BUNDLE"
setup_bundle "$DEBIAN_IMAGE" "$DEBIAN_BUNDLE"
}
function teardown_running_container() {
+1
View File
@@ -48,6 +48,7 @@ function teardown() {
.hooks |= . + {"createRuntime": [{"path": "/bin/sh", "args": ["/bin/sh", "-c", $create_runtime_hook]}]} |
.hooks |= . + {"createContainer": [{"path": "/bin/sh", "args": ["/bin/sh", "-c", $create_container_hook]}]} |
.hooks |= . + {"startContainer": [{"path": "/bin/sh", "args": ["/bin/sh", "-c", "ldconfig"]}]} |
.root.readonly |= false |
.process.args = ["/bin/sh", "-c", "ldconfig -p | grep librunc"]' "$DEBIAN_BUNDLE"/config.json)
echo "${CONFIG}" >config.json
-44
View File
@@ -1,44 +0,0 @@
#!/bin/bash
get_busybox() {
case $(go env GOARCH) in
arm64)
echo 'https://github.com/docker-library/busybox/raw/dist-arm64v8/stable/glibc/busybox.tar.xz'
;;
*)
echo 'https://github.com/docker-library/busybox/raw/dist-amd64/stable/glibc/busybox.tar.xz'
;;
esac
}
get_hello() {
case $(go env GOARCH) in
arm64)
echo 'hello-world-aarch64.tar'
;;
*)
echo 'hello-world.tar'
;;
esac
}
get_and_extract_debian() {
tmp=$(mktemp -d)
cd "$tmp"
debian="debian:3.11.6"
case $(go env GOARCH) in
arm64)
skopeo copy docker://arm64v8/debian:buster "oci:$debian"
;;
*)
skopeo copy docker://amd64/debian:buster "oci:$debian"
;;
esac
args="$([ -z "${ROOTLESS_TESTPATH+x}" ] && echo "--rootless")"
umoci unpack $args --image "$debian" "$1"
cd -
rm -rf "$tmp"
}
+2
View File
@@ -0,0 +1,2 @@
busybox-*.tar.xz
debian-*.tar.xz
+26 -15
View File
@@ -19,12 +19,16 @@
# a new feature, please match the existing style. Add an entry to $ALL_FEATURES,
# and add an enable_* and disable_* hook.
set -e -u -o pipefail
: "${RUNC_USE_SYSTEMD:=}"
: "${ROOTLESS_TESTPATH:=}"
ALL_FEATURES=("idmap" "cgroup")
# cgroup is managed by systemd when RUNC_USE_SYSTEMD is set
if [[ -n "${RUNC_USE_SYSTEMD}" ]]; then
ALL_FEATURES=("idmap")
fi
ROOT="$(readlink -f "$(dirname "${BASH_SOURCE}")/..")"
ROOT="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/..")"
# FEATURE: Opportunistic new{uid,gid}map support, allowing a rootless container
# to be set up with the usage of helper setuid binaries.
@@ -54,14 +58,12 @@ function enable_idmap() {
# Create a directory owned by $AUX_UID inside container, to be used
# by a test case in cwd.bats. This setup can't be done by the test itself,
# as it needs root for chown.
set -e
export AUX_UID=1024
AUX_DIR="$(mktemp -d)"
# 1000 is linux.uidMappings.containerID value,
# as set by runc_rootless_idmap
chown "$((ROOTLESS_UIDMAP_START - 1000 + AUX_UID))" "$AUX_DIR"
export AUX_DIR
set +e
}
function disable_idmap() {
@@ -75,10 +77,12 @@ function disable_idmap() {
# Deactivate new{uid,gid}map helpers. setuid is preserved with mv(1).
[ -e /usr/bin/newuidmap ] && mv /usr/bin/{,unused-}newuidmap
[ -e /usr/bin/newgidmap ] && mv /usr/bin/{,unused-}newgidmap
return 0
}
function cleanup() {
if [ -n "$AUX_DIR" ]; then
if [ -v AUX_DIR ]; then
rmdir "$AUX_DIR"
unset AUX_DIX
fi
@@ -90,7 +94,8 @@ function cleanup() {
# List of cgroups. We handle name= cgroups as well as combined
# (comma-separated) cgroups and correctly split and/or strip them.
ALL_CGROUPS=($(cat /proc/self/cgroup | cut -d: -f2 | sed -E '{s/^name=//;s/,/\n/;/^$/D}'))
# shellcheck disable=SC2207
ALL_CGROUPS=($(cut -d: -f2 </proc/self/cgroup | sed -E '{s/^name=//;s/,/\n/;/^$/D}'))
CGROUP_MOUNT="/sys/fs/cgroup"
CGROUP_PATH="/runc-cgroups-integration-test"
@@ -114,7 +119,8 @@ function enable_cgroup() {
if [[ -e "$CGROUP_MOUNT/cgroup.controllers" ]]; then
# Enable controllers. Some controller (e.g. memory) may fail on containerized environment.
set -x
for f in $(cat "$CGROUP_MOUNT/cgroup.controllers"); do echo +$f >"$CGROUP_MOUNT/cgroup.subtree_control"; done
# shellcheck disable=SC2013
for f in $(cat "$CGROUP_MOUNT/cgroup.controllers"); do echo "+$f" >"$CGROUP_MOUNT/cgroup.subtree_control"; done
set +x
# Create the cgroup.
mkdir -p "$CGROUP_MOUNT/$CGROUP_PATH"
@@ -133,6 +139,8 @@ function disable_cgroup() {
done
# cgroup v2
[ -d "$CGROUP_MOUNT/$CGROUP_PATH" ] && rmdir "$CGROUP_MOUNT/$CGROUP_PATH"
return 0
}
# Create a powerset of $ALL_FEATURES (the set of all subsets of $ALL_FEATURES).
@@ -140,35 +148,38 @@ function disable_cgroup() {
# feature knobs this shouldn't take too long -- but the number of tested
# combinations is O(2^n)).
function powerset() {
eval printf '%s' $(printf '{,%s+}' "$@"):
eval printf '%s' "$(printf '{,%s+}' "$@")":
}
features_powerset="$(powerset "${ALL_FEATURES[@]}")"
# Make sure we have container images downloaded, as otherwise
# rootless user won't be able to write to $TESTDATA.
"$ROOT"/tests/integration/get-images.sh >/dev/null
# Iterate over the powerset of all features.
IFS=:
idx=0
for enabled_features in $features_powerset; do
idx="$(($idx + 1))"
echo "[$(printf '%.2d' "$idx")] run rootless tests ... (${enabled_features%%+})"
((++idx))
printf "[%.2d] run rootless tests ... (${enabled_features%%+})\n" "$idx"
unset IFS
for feature in "${ALL_FEATURES[@]}"; do
hook_func="disable_$feature"
grep -E "(^|\+)$feature(\+|$)" <<<$enabled_features &>/dev/null && hook_func="enable_$feature"
grep -E "(^|\+)$feature(\+|$)" <<<"$enabled_features" &>/dev/null && hook_func="enable_$feature"
"$hook_func"
done
# Run the test suite!
set -e
echo path: $PATH
echo "path: $PATH"
export ROOTLESS_FEATURES="$enabled_features"
if [[ -n "${RUNC_USE_SYSTEMD}" ]]; then
# We use `ssh rootless@localhost` instead of `sudo -u rootless` for creating systemd user session.
# Alternatively we could use `machinectl shell`, but it is known not to work well on SELinux-enabled hosts as of April 2020:
# https://bugzilla.redhat.com/show_bug.cgi?id=1788616
ssh -t -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $HOME/rootless.key rootless@localhost -- PATH="$PATH" RUNC_USE_SYSTEMD="$RUNC_USE_SYSTEMD" bats -t "$ROOT/tests/integration$ROOTLESS_TESTPATH"
ssh -t -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$HOME/rootless.key" rootless@localhost -- PATH="$PATH" RUNC_USE_SYSTEMD="$RUNC_USE_SYSTEMD" bats -t "$ROOT/tests/integration$ROOTLESS_TESTPATH"
else
sudo -HE -u rootless PATH="$PATH" $(which bats) -t "$ROOT/tests/integration$ROOTLESS_TESTPATH"
sudo -HE -u rootless PATH="$PATH" "$(which bats)" -t "$ROOT/tests/integration$ROOTLESS_TESTPATH"
fi
set +e
cleanup
done