Files
runc/tests/integration/run.bats
T
Aleksa Sarai 515f09f7b1 dmz: use overlayfs to write-protect /proc/self/exe if possible
Commit b999376fb2 ("nsenter: cloned_binary: remove bindfd logic
entirely") removed the read-only bind-mount logic from our cloned binary
code because it wasn't really safe because a container with
CAP_SYS_ADMIN could remove the MS_RDONLY bit and get write access to
/proc/self/exe (even with user namespaces this could've been an issue
because it's not clear if the flags are locked).

However, copying a binary does seem to have a minor performance impact.
The only way to have no performance impact would be for the kernel to
block these write attempts, but barring that we could try to reduce the
overhead by coming up with a mount that cannot have it's read-only bits
cleared.

The "simplest" solution is to create a temporary overlayfs using
fsopen(2) which uses the directory where runc exists as a lowerdir,
ensuring that the container cannot access the underlying file -- and we
don't have to do any copies.

While fsopen(2) is not free because mount namespace cloning is usually
expensive (and so it seems like the difference would be marginal), some
basic performance testing seems to indicate there is a ~60% improvement
doing it this way and that it has effectively no overhead even when
compared to just using /proc/self/exe directly:

  % hyperfine --warmup 50 \
  >           "./runc-noclone run -b bundle ctr" \
  >           "./runc-overlayfs run -b bundle ctr" \
  >           "./runc-memfd run -b bundle ctr"

  Benchmark 1: ./runc-noclone run -b bundle ctr
    Time (mean ± σ):      13.7 ms ±   0.9 ms    [User: 6.0 ms, System: 10.9 ms]
    Range (min … max):    11.3 ms …  16.1 ms    184 runs

  Benchmark 2: ./runc-overlayfs run -b bundle ctr
    Time (mean ± σ):      13.9 ms ±   0.9 ms    [User: 6.2 ms, System: 10.8 ms]
    Range (min … max):    11.8 ms …  16.0 ms    180 runs

  Benchmark 3: ./runc-memfd run -b bundle ctr
    Time (mean ± σ):      22.6 ms ±   1.3 ms    [User: 5.7 ms, System: 20.7 ms]
    Range (min … max):    19.9 ms …  26.5 ms    114 runs

  Summary
    ./runc-noclone run -b bundle ctr ran
      1.01 ± 0.09 times faster than ./runc-overlayfs run -b bundle ctr
      1.65 ± 0.15 times faster than ./runc-memfd run -b bundle ctr

Signed-off-by: Aleksa Sarai <cyphar@cyphar.com>
2024-10-20 21:35:09 +11:00

270 lines
8.0 KiB
Bash

#!/usr/bin/env bats
load helpers
function setup() {
setup_busybox
update_config '.process.args = ["/bin/echo", "Hello World"]'
}
function teardown() {
teardown_bundle
}
@test "runc run" {
runc run test_hello
[ "$status" -eq 0 ]
runc state test_hello
[ "$status" -ne 0 ]
}
@test "runc run --keep" {
runc run --keep test_run_keep
[ "$status" -eq 0 ]
testcontainer test_run_keep stopped
runc state test_run_keep
[ "$status" -eq 0 ]
runc delete test_run_keep
runc state test_run_keep
[ "$status" -ne 0 ]
}
@test "runc run --keep (check cgroup exists)" {
# for systemd driver, the unit's cgroup path will be auto removed if container's all processes exited
requires no_systemd
[ $EUID -ne 0 ] && requires rootless_cgroup
set_cgroups_path
runc run --keep test_run_keep
[ "$status" -eq 0 ]
testcontainer test_run_keep stopped
runc state test_run_keep
[ "$status" -eq 0 ]
# check that cgroup exists
check_cgroup_value "pids.max" "max"
runc delete test_run_keep
runc state test_run_keep
[ "$status" -ne 0 ]
}
@test "runc run [hostname domainname]" {
update_config ' .process.args |= ["sh"]
| .hostname = "myhostname"
| .domainname= "mydomainname"'
runc run -d --console-socket "$CONSOLE_SOCKET" test_utc
[ "$status" -eq 0 ]
# test hostname
runc exec test_utc hostname
[ "$status" -eq 0 ]
[[ "${lines[0]}" == *'myhostname'* ]]
# test domainname
runc exec test_utc cat /proc/sys/kernel/domainname
[ "$status" -eq 0 ]
[[ "${lines[0]}" == *'mydomainname'* ]]
}
# https://github.com/opencontainers/runc/issues/3952
@test "runc run with tmpfs" {
requires root
chmod 'a=rwx,ug+s,+t' rootfs/tmp # set all bits
mode=$(stat -c %A rootfs/tmp)
# shellcheck disable=SC2016
update_config '.process.args = ["sh", "-c", "stat -c %A /tmp"]'
update_config '.mounts += [{"destination": "/tmp", "type": "tmpfs", "source": "tmpfs", "options":["noexec","nosuid","nodev","rprivate"]}]'
runc run test_tmpfs
[ "$status" -eq 0 ]
[ "${lines[0]}" = "$mode" ]
}
@test "runc run with tmpfs perms" {
# shellcheck disable=SC2016
update_config '.process.args = ["sh", "-c", "stat -c %a /tmp/test"]'
update_config '.mounts += [{"destination": "/tmp/test", "type": "tmpfs", "source": "tmpfs", "options": ["mode=0444"]}]'
# Directory is to be created by runc.
runc run test_tmpfs
[ "$status" -eq 0 ]
[ "${lines[0]}" = "444" ]
# Run a 2nd time with the pre-existing directory.
# Ref: https://github.com/opencontainers/runc/issues/3911
runc run test_tmpfs
[ "$status" -eq 0 ]
[ "${lines[0]}" = "444" ]
# Existing directory, custom perms, no mode on the mount,
# so it should use the directory's perms.
update_config '.mounts[-1].options = []'
chmod 0710 rootfs/tmp/test
# shellcheck disable=SC2016
runc run test_tmpfs
[ "$status" -eq 0 ]
[ "${lines[0]}" = "710" ]
# Add back the mode on the mount, and it should use that instead.
# Just for fun, use different perms than was used earlier.
# shellcheck disable=SC2016
update_config '.mounts[-1].options = ["mode=0410"]'
runc run test_tmpfs
[ "$status" -eq 0 ]
[ "${lines[0]}" = "410" ]
}
@test "RUNC_DMZ=true runc run [runc-dmz]" {
RUNC_DMZ=true runc --debug run test_hello
[ "$status" -eq 0 ]
[[ "$output" = *"Hello World"* ]]
# We use runc-dmz if we can.
[[ "$output" = *"runc-dmz: using runc-dmz"* ]]
}
@test "RUNC_DMZ=true runc run [cap_sys_ptrace -> /proc/self/exe clone]" {
# Add CAP_SYS_PTRACE to the bounding set, the minimum needed to indicate a
# container process _could_ get CAP_SYS_PTRACE.
update_config '.process.capabilities.bounding += ["CAP_SYS_PTRACE"]'
RUNC_DMZ=true runc --debug run test_hello
[ "$status" -eq 0 ]
[[ "$output" = *"Hello World"* ]]
if [ "$EUID" -ne 0 ] && is_kernel_gte 4.10; then
# For Linux 4.10 and later, rootless containers will use runc-dmz
# because they are running in a user namespace. See isDmzBinarySafe().
[[ "$output" = *"runc-dmz: using runc-dmz"* ]]
else
# If the container has CAP_SYS_PTRACE and is not rootless, we use
# /proc/self/exe cloning.
[[ "$output" = *"runc-dmz: using /proc/self/exe clone"* ]]
fi
}
@test "runc run [/proc/self/exe clone]" {
runc --debug run test_hello
[ "$status" -eq 0 ]
[[ "$output" = *"Hello World"* ]]
[[ "$output" = *"runc-dmz: using /proc/self/exe clone"* ]]
# runc will use fsopen("overlay") if it can.
if can_fsopen overlay; then
[[ "$output" = *"runc-dmz: using overlayfs for sealed /proc/self/exe"* ]]
fi
}
@test "runc run [joining existing container namespaces]" {
requires timens
# Create a detached container with the namespaces we want. We notably want
# to include both userns and timens, which require config-related
# configuration.
if [ $EUID -eq 0 ]; then
update_config '.linux.namespaces += [{"type": "user"}]
| .linux.uidMappings += [{"containerID": 0, "hostID": 100000, "size": 100}]
| .linux.gidMappings += [{"containerID": 0, "hostID": 200000, "size": 200}]'
remap_rootfs
fi
update_config '.linux.namespaces += [{"type": "time"}]
| .linux.timeOffsets = {
"monotonic": { "secs": 7881, "nanosecs": 2718281 },
"boottime": { "secs": 1337, "nanosecs": 3141519 }
}'
update_config '.process.args = ["sleep", "infinity"]'
runc run -d --console-socket "$CONSOLE_SOCKET" target_ctr
[ "$status" -eq 0 ]
# Modify our container's configuration such that it is just going to
# inherit all of the namespaces of the target container.
#
# NOTE: We cannot join the mount namespace of another container because of
# some quirks of the runtime-spec. In particular, we MUST pivot_root into
# root.path and root.path MUST be set in the config, so runc cannot just
# ignore root.path when joining namespaces (and root.path doesn't exist
# inside root.path, for obvious reasons).
#
# We could hack around this (create a copy of the rootfs inside the rootfs,
# or use a simpler mount namespace target), but those wouldn't be similar
# tests to the other namespace joining tests.
target_pid="$(__runc state target_ctr | jq .pid)"
update_config '.linux.namespaces |= map_values(.path = if .type == "mount" then "" else "/proc/'"$target_pid"'/ns/" + ({"network": "net", "mount": "mnt"}[.type] // .type) end)'
# Remove the userns and timens configuration (they cannot be changed).
update_config '.linux |= (del(.uidMappings) | del(.gidMappings) | del(.timeOffsets))'
runc run -d --console-socket "$CONSOLE_SOCKET" attached_ctr
[ "$status" -eq 0 ]
# Make sure there are two sleep processes in our container.
runc exec attached_ctr ps aux
[ "$status" -eq 0 ]
run -0 grep "sleep infinity" <<<"$output"
[ "${#lines[@]}" -eq 2 ]
# ... that the userns mappings are the same...
runc exec attached_ctr cat /proc/self/uid_map
[ "$status" -eq 0 ]
if [ $EUID -eq 0 ]; then
grep -E '^\s+0\s+100000\s+100$' <<<"$output"
else
grep -E '^\s+0\s+'$EUID'\s+1$' <<<"$output"
fi
runc exec attached_ctr cat /proc/self/gid_map
[ "$status" -eq 0 ]
if [ $EUID -eq 0 ]; then
grep -E '^\s+0\s+200000\s+200$' <<<"$output"
else
grep -E '^\s+0\s+'$EUID'\s+1$' <<<"$output"
fi
# ... as well as the timens offsets.
runc exec attached_ctr cat /proc/self/timens_offsets
grep -E '^monotonic\s+7881\s+2718281$' <<<"$output"
grep -E '^boottime\s+1337\s+3141519$' <<<"$output"
}
@test "RUNC_DMZ=true runc run [exec error]" {
cat <<EOF >rootfs/run.sh
#!/mmnnttbb foo bar
sh
EOF
chmod +x rootfs/run.sh
update_config '.process.args = [ "/run.sh" ]'
RUNC_DMZ=true runc run test_hello
# Ensure that the output contains the right error message. For runc-dmz, both
# nolibc and libc have the same formatting string (but libc will print the
# errno description rather than just the number), and for runc_nodmz the error
# message from Go starts with the same string.
[ "$status" -ne 0 ]
[[ "$output" = *"exec /run.sh: "* ]]
}
@test "runc run [execve error]" {
cat <<EOF >rootfs/run.sh
#!/mmnnttbb foo bar
sh
EOF
chmod +x rootfs/run.sh
update_config '.process.args = [ "/run.sh" ]'
runc run test_hello
[ "$status" -ne 0 ]
# After the sync socket closed, we should not send error to parent
# process, or else we will get a unnecessary error log(#4171).
[ ${#lines[@]} -eq 1 ]
[[ ${lines[0]} = "exec /run.sh: no such file or directory" ]]
}