From 41670e21f0f36390ee9cbd5b18590e66f96ea414 Mon Sep 17 00:00:00 2001
From: Kir Kolyshkin <kolyshkin@gmail.com>
Date: Thu, 21 Jan 2021 15:25:52 -0800
Subject: [PATCH] tests/int: rework/simplify setup and teardown

1. Get rid of fixed ROOT, *_BUNDLE, and CONSOLE_SOCKET dirs.
   Now they are temporary directories created in setup_bundle.

2. Automate containers cleanup: instead of having to specify all
   containers to be removed, list and destroy everything (which is
   now possible since every test case has its own unique root).

3. Randomize cgroup paths so two tests running in parallel won't
   use the same cgroup.

Now it's theoretically possible to execute tests in parallel.
Practically it's not possible yet because bats uses GNU parallel,
which do not provide a terminal for whatever it executes, and
many runc tests (all those that run containers with terminal:
true) needs a tty. This may possibly be addressed later.

Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>
---
 tests/integration/README.md           |  5 +-
 tests/integration/cgroups.bats        |  7 +--
 tests/integration/checkpoint.bats     |  4 +-
 tests/integration/create.bats         |  3 +-
 tests/integration/cwd.bats            |  3 +-
 tests/integration/debug.bats          |  3 +-
 tests/integration/delete.bats         |  4 +-
 tests/integration/dev.bats            |  4 +-
 tests/integration/events.bats         |  5 +-
 tests/integration/exec.bats           |  3 +-
 tests/integration/helpers.bash        | 81 +++++++++++----------------
 tests/integration/hooks.bats          |  3 +-
 tests/integration/kill.bats           |  3 +-
 tests/integration/list.bats           | 12 ++--
 tests/integration/mask.bats           |  3 +-
 tests/integration/mounts.bats         |  3 +-
 tests/integration/no_pivot.bats       |  3 +-
 tests/integration/pause.bats          |  3 +-
 tests/integration/ps.bats             |  3 +-
 tests/integration/root.bats           |  9 ++-
 tests/integration/seccomp.bats        |  3 +-
 tests/integration/spec.bats           |  2 +-
 tests/integration/start.bats          |  3 +-
 tests/integration/start_detached.bats |  3 +-
 tests/integration/start_hello.bats    |  3 +-
 tests/integration/state.bats          |  3 +-
 tests/integration/tty.bats            |  3 +-
 tests/integration/umask.bats          |  3 +-
 tests/integration/update.bats         |  5 +-
 29 files changed, 69 insertions(+), 123 deletions(-)

diff --git a/tests/integration/README.md b/tests/integration/README.md
index 2ce93907b..5dcba0020 100644
--- a/tests/integration/README.md
+++ b/tests/integration/README.md
@@ -61,15 +61,12 @@ load helpers
 
 # setup is called at the beginning of every test.
 function setup() {
-  # see functions teardown_hello and setup_hello in helpers.bash, used to
-  # create a pristine environment for running your tests
-  teardown_hello
   setup_hello
 }
 
 # teardown is called at the end of every test.
 function teardown() {
-  teardown_hello
+  teardown_bundle
 }
 
 @test "this is a simple test" {
diff --git a/tests/integration/cgroups.bats b/tests/integration/cgroups.bats
index 4ca07ab4d..80f9a53f4 100644
--- a/tests/integration/cgroups.bats
+++ b/tests/integration/cgroups.bats
@@ -3,15 +3,10 @@
 load helpers
 
 function teardown() {
-	teardown_running_container test_cgroups_kmem
-	teardown_running_container test_cgroups_permissions
-	teardown_running_container test_cgroups_group
-	teardown_running_container test_cgroups_unified
-	teardown_busybox
+	teardown_bundle
 }
 
 function setup() {
-	teardown
 	setup_busybox
 }
 
diff --git a/tests/integration/checkpoint.bats b/tests/integration/checkpoint.bats
index e45538380..ccfbe0b7d 100644
--- a/tests/integration/checkpoint.bats
+++ b/tests/integration/checkpoint.bats
@@ -6,13 +6,11 @@ function setup() {
 	# XXX: currently criu require root containers.
 	requires criu root
 
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
-	teardown_running_container test_busybox_restore
+	teardown_bundle
 }
 
 function setup_pipes() {
diff --git a/tests/integration/create.bats b/tests/integration/create.bats
index 663efe2c9..afa9f9dfb 100644
--- a/tests/integration/create.bats
+++ b/tests/integration/create.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "runc create" {
diff --git a/tests/integration/cwd.bats b/tests/integration/cwd.bats
index 6fe533a91..784c70bc9 100644
--- a/tests/integration/cwd.bats
+++ b/tests/integration/cwd.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 # Test case for https://github.com/opencontainers/runc/pull/2086
diff --git a/tests/integration/debug.bats b/tests/integration/debug.bats
index a92b90139..16f1b2fb0 100644
--- a/tests/integration/debug.bats
+++ b/tests/integration/debug.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_hello
 	setup_hello
 }
 
 function teardown() {
-	teardown_hello
+	teardown_bundle
 }
 
 @test "global --debug" {
diff --git a/tests/integration/delete.bats b/tests/integration/delete.bats
index c7cd4e26b..ccb9108c9 100644
--- a/tests/integration/delete.bats
+++ b/tests/integration/delete.bats
@@ -3,13 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
-	teardown_running_container testbusyboxdelete
+	teardown_bundle
 }
 
 @test "runc delete" {
diff --git a/tests/integration/dev.bats b/tests/integration/dev.bats
index 11c53c955..3f486f1c9 100644
--- a/tests/integration/dev.bats
+++ b/tests/integration/dev.bats
@@ -3,13 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
-	teardown_running_container test_dev
+	teardown_bundle
 }
 
 @test "runc run [redundant default /dev/tty]" {
diff --git a/tests/integration/events.bats b/tests/integration/events.bats
index cb004118a..9d28420e5 100644
--- a/tests/integration/events.bats
+++ b/tests/integration/events.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "events --stats" {
@@ -48,7 +47,7 @@ function test_events() {
 	#    test_busybox container which causes the event logger to exit.
 	(
 		retry 10 "$retry_every" grep -q test_busybox events.log
-		teardown_running_container test_busybox
+		__runc delete -f test_busybox
 	) &
 	wait # for both subshells to finish
 
diff --git a/tests/integration/exec.bats b/tests/integration/exec.bats
index 4aa059dbc..b7c9becbf 100644
--- a/tests/integration/exec.bats
+++ b/tests/integration/exec.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "runc exec" {
diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash
index e1092b4f6..a35ec62d7 100644
--- a/tests/integration/helpers.bash
+++ b/tests/integration/helpers.bash
@@ -20,11 +20,6 @@ RECVTTY="${INTEGRATION_ROOT}/../../contrib/cmd/recvtty/recvtty"
 # Test data path.
 TESTDATA="${INTEGRATION_ROOT}/testdata"
 
-# Destinations for test containers.
-BUSYBOX_BUNDLE="$BATS_RUN_TMPDIR/busyboxtest"
-HELLO_BUNDLE="$BATS_RUN_TMPDIR/hello-world"
-DEBIAN_BUNDLE="$BATS_RUN_TMPDIR/debiantest"
-
 # CRIU PATH
 CRIU="$(which criu 2>/dev/null || true)"
 
@@ -34,12 +29,6 @@ KERNEL_MAJOR="${KERNEL_VERSION%%.*}"
 KERNEL_MINOR="${KERNEL_VERSION#$KERNEL_MAJOR.}"
 KERNEL_MINOR="${KERNEL_MINOR%%.*}"
 
-# Root state path.
-ROOT=$(mktemp -d "$BATS_RUN_TMPDIR/runc.XXXXXX")
-
-# Path to console socket.
-CONSOLE_SOCKET="$BATS_RUN_TMPDIR/console.sock"
-
 # Check if we're in rootless mode.
 ROOTLESS=$(id -u)
 
@@ -55,7 +44,7 @@ function runc() {
 
 # Raw wrapper for runc.
 function __runc() {
-	"$RUNC" ${RUNC_USE_SYSTEMD+--systemd-cgroup} --root "$ROOT" "$@"
+	"$RUNC" ${RUNC_USE_SYSTEMD+--systemd-cgroup} --root "$ROOT/state" "$@"
 }
 
 # Wrapper for runc spec, which takes only one argument (the bundle path).
@@ -123,18 +112,19 @@ function init_cgroup_paths() {
 	# init once
 	test -n "$CGROUP_UNIFIED" && return
 
+	local rnd="$RANDOM"
 	if [ -n "${RUNC_USE_SYSTEMD}" ]; then
-		SD_UNIT_NAME="runc-cgroups-integration-test.scope"
+		SD_UNIT_NAME="runc-cgroups-integration-test-${rnd}.scope"
 		if [ $(id -u) = "0" ]; then
 			REL_CGROUPS_PATH="/machine.slice/$SD_UNIT_NAME"
-			OCI_CGROUPS_PATH="machine.slice:runc-cgroups:integration-test"
+			OCI_CGROUPS_PATH="machine.slice:runc-cgroups:integration-test-${rnd}"
 		else
 			REL_CGROUPS_PATH="/user.slice/user-$(id -u).slice/user@$(id -u).service/machine.slice/$SD_UNIT_NAME"
 			# OCI path doesn't contain "/user.slice/user-$(id -u).slice/user@$(id -u).service/" prefix
-			OCI_CGROUPS_PATH="machine.slice:runc-cgroups:integration-test"
+			OCI_CGROUPS_PATH="machine.slice:runc-cgroups:integration-test-${rnd}"
 		fi
 	else
-		REL_CGROUPS_PATH="/runc-cgroups-integration-test/test-cgroup"
+		REL_CGROUPS_PATH="/runc-cgroups-integration-test/test-cgroup-${rnd}"
 		OCI_CGROUPS_PATH=$REL_CGROUPS_PATH
 	fi
 
@@ -428,28 +418,38 @@ function testcontainer() {
 }
 
 function setup_recvtty() {
+	[ -z "$ROOT" ] && return 1 # must not be called without ROOT set
+	local dir="$ROOT/tty"
+
+	mkdir $dir
+	export CONSOLE_SOCKET="$dir/sock"
+
 	# We need to start recvtty in the background, so we double fork in the shell.
-	("$RECVTTY" --pid-file "$BATS_RUN_TMPDIR/recvtty.pid" --mode null "$CONSOLE_SOCKET" &) &
+	("$RECVTTY" --pid-file "$dir/pid" --mode null "$CONSOLE_SOCKET" &) &
 }
 
 function teardown_recvtty() {
+	[ -z "$ROOT" ] && return 0 # nothing to teardown
+	local dir="$ROOT/tty"
+
 	# When we kill recvtty, the container will also be killed.
-	if [ -f "$BATS_RUN_TMPDIR/recvtty.pid" ]; then
-		kill -9 $(cat "$BATS_RUN_TMPDIR/recvtty.pid")
+	if [ -f "$dir/pid" ]; then
+		kill -9 $(cat "$dir/pid")
 	fi
 
 	# Clean up the files that might be left over.
-	rm -f "$BATS_RUN_TMPDIR/recvtty.pid"
-	rm -f "$CONSOLE_SOCKET"
+	rm -rf "$dir"
 }
 
 function setup_bundle() {
 	local image="$1"
-	local bundle="$2"
+
+	# Root for various container directories (state, tty, bundle).
+	export ROOT=$(mktemp -d "$BATS_RUN_TMPDIR/runc.XXXXXX")
+	mkdir -p "$ROOT/state" "$ROOT/bundle/rootfs"
 
 	setup_recvtty
-	mkdir -p "$bundle"/rootfs
-	cd "$bundle"
+	cd "$ROOT/bundle"
 
 	tar --exclude './dev/*' -C rootfs -xf "$image"
 
@@ -457,39 +457,26 @@ function setup_bundle() {
 }
 
 function setup_busybox() {
-	setup_bundle "$BUSYBOX_IMAGE" "$BUSYBOX_BUNDLE"
+	setup_bundle "$BUSYBOX_IMAGE"
 }
 
 function setup_hello() {
-	setup_bundle "$HELLO_IMAGE" "$HELLO_BUNDLE"
+	setup_bundle "$HELLO_IMAGE"
 	update_config '(.. | select(.? == "sh")) |= "/hello"'
 }
 
 function setup_debian() {
-	setup_bundle "$DEBIAN_IMAGE" "$DEBIAN_BUNDLE"
+	setup_bundle "$DEBIAN_IMAGE"
 }
 
-function teardown_running_container() {
-	__runc delete -f "$1"
-}
+function teardown_bundle() {
+	[ -z "$ROOT" ] && return 0 # nothing to teardown
 
-function teardown_busybox() {
 	cd "$INTEGRATION_ROOT"
 	teardown_recvtty
-	teardown_running_container test_busybox
-	rm -f -r "$BUSYBOX_BUNDLE"
-}
-
-function teardown_hello() {
-	cd "$INTEGRATION_ROOT"
-	teardown_recvtty
-	teardown_running_container test_hello
-	rm -f -r "$HELLO_BUNDLE"
-}
-
-function teardown_debian() {
-	cd "$INTEGRATION_ROOT"
-	teardown_recvtty
-	teardown_running_container test_debian
-	rm -f -r "$DEBIAN_BUNDLE"
+	local ct
+	for ct in $(__runc list -q); do
+		__runc delete -f "$ct"
+	done
+	rm -rf "$ROOT"
 }
diff --git a/tests/integration/hooks.bats b/tests/integration/hooks.bats
index bc11e09c1..0c57bbc76 100644
--- a/tests/integration/hooks.bats
+++ b/tests/integration/hooks.bats
@@ -5,7 +5,6 @@ load helpers
 function setup() {
 	requires root no_systemd
 
-	teardown
 	setup_debian
 	# CR = CreateRuntime, CC = CreataContainer
 	HOOKLIBCR=librunc-hooks-create-runtime.so
@@ -19,7 +18,7 @@ function teardown() {
 		umount "$LIBPATH"/$HOOKLIBCC.1.0.0 &>/dev/null || true
 		rm -f $HOOKLIBCR.1.0.0 $HOOKLIBCC.1.0.0
 	fi
-	teardown_debian
+	teardown_bundle
 }
 
 @test "runc run (hooks library tests)" {
diff --git a/tests/integration/kill.bats b/tests/integration/kill.bats
index b324683df..590ddd5ef 100644
--- a/tests/integration/kill.bats
+++ b/tests/integration/kill.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "kill detached busybox" {
diff --git a/tests/integration/list.bats b/tests/integration/list.bats
index 97d6028ba..9d25dfd36 100644
--- a/tests/integration/list.bats
+++ b/tests/integration/list.bats
@@ -3,20 +3,16 @@
 load helpers
 
 function setup() {
-	unset ALT_ROOT
-	teardown
 	setup_busybox
-	ALT_ROOT=$(mktemp -d "$BATS_RUN_TMPDIR/runc-2.XXXXXX")
+	ALT_ROOT="$ROOT/alt"
+	mkdir -p "$ALT_ROOT/state"
 }
 
 function teardown() {
 	if [ -n "$ALT_ROOT" ]; then
-		ROOT="$ALT_ROOT" teardown_running_container test_box1
-		ROOT="$ALT_ROOT" teardown_running_container test_box2
-		ROOT="$ALT_ROOT" teardown_running_container test_box3
-		rm -rf "$ALT_ROOT"
+		ROOT="$ALT_ROOT" teardown_bundle
 	fi
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "list" {
diff --git a/tests/integration/mask.bats b/tests/integration/mask.bats
index 0786c2ad6..b5f296756 100644
--- a/tests/integration/mask.bats
+++ b/tests/integration/mask.bats
@@ -3,7 +3,6 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 
 	# Create fake rootfs.
@@ -15,7 +14,7 @@ function setup() {
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "mask paths [file]" {
diff --git a/tests/integration/mounts.bats b/tests/integration/mounts.bats
index a4b9f296e..f21136b60 100644
--- a/tests/integration/mounts.bats
+++ b/tests/integration/mounts.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "runc run [bind mount]" {
diff --git a/tests/integration/no_pivot.bats b/tests/integration/no_pivot.bats
index 844a0ca57..4baf0bf2d 100644
--- a/tests/integration/no_pivot.bats
+++ b/tests/integration/no_pivot.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "runc run --no-pivot must not expose bare /proc" {
diff --git a/tests/integration/pause.bats b/tests/integration/pause.bats
index 217d78168..c2daa1f46 100644
--- a/tests/integration/pause.bats
+++ b/tests/integration/pause.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "runc pause and resume" {
diff --git a/tests/integration/ps.bats b/tests/integration/ps.bats
index b611a3501..c28abfc2d 100644
--- a/tests/integration/ps.bats
+++ b/tests/integration/ps.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "ps" {
diff --git a/tests/integration/root.bats b/tests/integration/root.bats
index 127e5a9ab..53bd1ce56 100644
--- a/tests/integration/root.bats
+++ b/tests/integration/root.bats
@@ -3,18 +3,17 @@
 load helpers
 
 function setup() {
-	unset ALT_ROOT
-	teardown
 	setup_busybox
-	ALT_ROOT=$(mktemp -d "$BATS_RUN_TMPDIR/runc-2.XXXXXX")
+	ALT_ROOT="$ROOT/alt"
+	mkdir -p "$ALT_ROOT/state"
 }
 
 function teardown() {
 	if [ -n "$ALT_ROOT" ]; then
-		ROOT=$ALT_ROOT teardown_running_container test_dotbox
+		ROOT=$ALT_ROOT __runc delete -f test_dotbox
 		rm -rf "$ALT_ROOT"
 	fi
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "global --root" {
diff --git a/tests/integration/seccomp.bats b/tests/integration/seccomp.bats
index 267c6837a..937f961a2 100644
--- a/tests/integration/seccomp.bats
+++ b/tests/integration/seccomp.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "runc run [seccomp -ENOSYS handling]" {
diff --git a/tests/integration/spec.bats b/tests/integration/spec.bats
index dbb022ace..b24c6b714 100644
--- a/tests/integration/spec.bats
+++ b/tests/integration/spec.bats
@@ -7,7 +7,7 @@ function setup() {
 }
 
 function teardown() {
-	teardown_hello
+	teardown_bundle
 }
 
 @test "spec generation cwd" {
diff --git a/tests/integration/start.bats b/tests/integration/start.bats
index 6a6e61684..1c844b251 100644
--- a/tests/integration/start.bats
+++ b/tests/integration/start.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "runc start" {
diff --git a/tests/integration/start_detached.bats b/tests/integration/start_detached.bats
index 37b520bc3..0f1c23326 100644
--- a/tests/integration/start_detached.bats
+++ b/tests/integration/start_detached.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "runc run detached" {
diff --git a/tests/integration/start_hello.bats b/tests/integration/start_hello.bats
index b30138bdf..5c2a66fdb 100644
--- a/tests/integration/start_hello.bats
+++ b/tests/integration/start_hello.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_hello
 	setup_hello
 }
 
 function teardown() {
-	teardown_hello
+	teardown_bundle
 }
 
 @test "runc run" {
diff --git a/tests/integration/state.bats b/tests/integration/state.bats
index 04026a52a..5e88a3e28 100644
--- a/tests/integration/state.bats
+++ b/tests/integration/state.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "state (kill + delete)" {
diff --git a/tests/integration/tty.bats b/tests/integration/tty.bats
index a1ede7bfa..8bbb47649 100644
--- a/tests/integration/tty.bats
+++ b/tests/integration/tty.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "runc run [stdin not a tty]" {
diff --git a/tests/integration/umask.bats b/tests/integration/umask.bats
index e8166cda2..fe9f9d637 100644
--- a/tests/integration/umask.bats
+++ b/tests/integration/umask.bats
@@ -3,12 +3,11 @@
 load helpers
 
 function setup() {
-	teardown_busybox
 	setup_busybox
 }
 
 function teardown() {
-	teardown_busybox
+	teardown_bundle
 }
 
 @test "umask" {
diff --git a/tests/integration/update.bats b/tests/integration/update.bats
index dd0adea72..049d9587e 100644
--- a/tests/integration/update.bats
+++ b/tests/integration/update.bats
@@ -4,13 +4,10 @@ load helpers
 
 function teardown() {
 	rm -f "$BATS_RUN_TMPDIR"/runc-cgroups-integration-test.json
-	teardown_running_container test_update
-	teardown_running_container test_update_rt
-	teardown_busybox
+	teardown_bundle
 }
 
 function setup() {
-	teardown
 	setup_busybox
 
 	set_cgroups_path