Merge pull request #4869 from cyphar/1.2-reset-cpu-affinity

[1.2] libct: reset CPU affinity by default
2026-07-11 06:03:57 +08:00 · 2025-08-28 12:27:08 +08:00
parent d5350ff31b 6983cc7ac1
commit 2f9d7aecf9
4 changed files with 201 additions and 6 deletions
@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ## [Unreleased 1.2.z]

+### Fixed
+ * Container processes will no longer inherit the CPU affinity of runc by
+   default. Instead, the default CPU affinity of container processes will be
+   the largest set of CPUs permitted by the container's cpuset cgroup and any
+   other system restrictions (such as isolated CPUs). (#4041, #4815, #4858)
+
 ## [1.2.6] - 2025-03-17

 > Hasta la victoria, siempre.
@@ -122,6 +122,46 @@ func (p *setnsProcess) signal(sig os.Signal) error {
 	return unix.Kill(p.pid(), s)
 }

+// tryResetCPUAffinity tries to reset the CPU affinity of the process
+// identified by pid to include all possible CPUs (notwithstanding cgroup
+// cpuset restrictions and isolated CPUs).
+func tryResetCPUAffinity(pid int) {
+	// When resetting the CPU affinity, we want to match the configured cgroup
+	// cpuset (or the default set of all CPUs, if no cpuset is configured)
+	// rather than some more restrictive affinity we were spawned in (such as
+	// one that may have been inherited from systemd). The cpuset cgroup used
+	// to reconfigure the cpumask automatically for joining processes, but
+	// kcommit da019032819a ("sched: Enforce user requested affinity") changed
+	// this behaviour in Linux 6.2.
+	//
+	// Parsing cpuset.cpus.effective is quite inefficient (and looking at
+	// things like /proc/stat would be wrong for most nested containers), but
+	// luckily sched_setaffinity(2) will implicitly:
+	//
+	//  * Clamp the cpumask so that it matches the current number of CPUs on
+	//    the system.
+	//  * Mask out any CPUs that are not a member of the target task's
+	//    configured cgroup cpuset.
+	//
+	// So we can just pass a very large array of set cpumask bits and the
+	// kernel will silently convert that to the correct value very cheaply.
+
+	// Ideally, we would just set the array to 0xFF...FF. Unfortunately, the
+	// size depends on the architecture. It is also a private newtype, so we
+	// can't use (^0) or generics since those require us to be able to name the
+	// type. However, we can just underflow the zero value instead.
+	// TODO: Once <https://golang.org/cl/698015> is merged, switch to that.
+	cpuset := unix.CPUSet{}
+	for i := range cpuset {
+		cpuset[i]-- // underflow to 0xFF..FF
+	}
+	if err := unix.SchedSetaffinity(pid, &cpuset); err != nil {
+		logrus.WithError(
+			os.NewSyscallError("sched_setaffinity", err),
+		).Warnf("resetting the CPU affinity of pid %d failed -- the container process may inherit runc's CPU affinity", pid)
+	}
+}
+
 func (p *setnsProcess) start() (retErr error) {
 	defer p.comm.closeParent()

@@ -184,6 +224,9 @@ func (p *setnsProcess) start() (retErr error) {
 			}
 		}
 	}
+	// Reset the CPU affinity after cgroups are configured to make sure it
+	// matches any configured cpuset.
+	tryResetCPUAffinity(p.pid())
 	if p.intelRdtPath != "" {
 		// if Intel RDT "resource control" filesystem path exists
 		_, err := os.Stat(p.intelRdtPath)
@@ -578,6 +621,9 @@ func (p *initProcess) start() (retErr error) {
 			return fmt.Errorf("unable to apply cgroup configuration: %w", err)
 		}
 	}
+	// Reset the CPU affinity after cgroups are configured to make sure it
+	// matches any configured cpuset.
+	tryResetCPUAffinity(p.pid())
 	if p.intelRdtManager != nil {
 		if err := p.intelRdtManager.Apply(p.pid()); err != nil {
 			return fmt.Errorf("unable to apply Intel RDT configuration: %w", err)
@@ -0,0 +1,127 @@
+#!/usr/bin/env bats
+# Exec CPU affinity tests. For more details, see:
+#  - https://github.com/opencontainers/runtime-spec/pull/1253
+
+load helpers
+
+INITIAL_CPU_MASK="$(grep -F Cpus_allowed_list: /proc/self/status | awk '{ print $2 }')"
+
+function setup() {
+	requires smp cgroups_cpuset
+	setup_busybox
+
+	echo "Initial CPU mask: $INITIAL_CPU_MASK" >&2
+	echo "---" >&2
+}
+
+function teardown() {
+	teardown_bundle
+}
+
+function first_cpu() {
+	sed 's/[-,].*//g' </sys/devices/system/cpu/online
+}
+
+@test "runc run [CPU affinity should reset]" {
+	# We need to use RUNC_CMDLINE since taskset requires a proper binary, not a
+	# bash function (which is what runc and __runc are).
+	setup_runc_cmdline
+
+	first="$(first_cpu)"
+
+	# Running without cpuset should result in an affinity for all CPUs.
+	update_config '.process.args = [ "/bin/grep", "-F", "Cpus_allowed_list:", "/proc/self/status" ]'
+	update_config 'del(.linux.resources.cpu)'
+	sane_run taskset -c "$first" "${RUNC_CMDLINE[@]}" run ctr
+	[ "$status" -eq 0 ]
+	[[ "$output" != $'Cpus_allowed_list:\t'"$first" ]]
+	[[ "$output" == $'Cpus_allowed_list:\t'"$INITIAL_CPU_MASK" ]]
+}
+
+@test "runc run [CPU affinity should reset to cgroup cpuset]" {
+	[ $EUID -ne 0 ] && requires rootless_cgroup
+	set_cgroups_path
+
+	# We need to use RUNC_CMDLINE since taskset requires a proper binary, not a
+	# bash function (which is what runc and __runc are).
+	setup_runc_cmdline
+
+	first="$(first_cpu)"
+	second="$((first + 1))" # Hacky; might not work in all environments.
+
+	# Running with a cpuset should result in an affinity that matches.
+	update_config '.process.args = [ "/bin/grep", "-F", "Cpus_allowed_list:", "/proc/self/status" ]'
+	update_config '.linux.resources.cpu = {"mems": "0", "cpus": "'"$first-$second"'"}'
+	sane_run taskset -c "$first" "${RUNC_CMDLINE[@]}" run ctr
+	[ "$status" -eq 0 ]
+	[[ "$output" != $'Cpus_allowed_list:\t'"$first" ]]
+	# XXX: For some reason, systemd-cgroup leads to us using the all-set
+	#      cpumask rather than the cpuset we configured?
+	[ -v RUNC_USE_SYSTEMD ] || [[ "$output" == $'Cpus_allowed_list:\t'"$first-$second" ]]
+
+	# Ditto for a cpuset that has no overlap with the original cpumask.
+	update_config '.linux.resources.cpu = {"mems": "0", "cpus": "'"$second"'"}'
+	sane_run taskset -c "$first" "${RUNC_CMDLINE[@]}" run ctr
+	[ "$status" -eq 0 ]
+	[[ "$output" != $'Cpus_allowed_list:\t'"$first" ]]
+	# XXX: For some reason, systemd-cgroup leads to us using the all-set
+	#      cpumask rather than the cpuset we configured?
+	[ -v RUNC_USE_SYSTEMD ] || [[ "$output" == $'Cpus_allowed_list:\t'"$second" ]]
+}
+
+@test "runc exec [default CPU affinity should reset]" {
+	# We need to use RUNC_CMDLINE since taskset requires a proper binary, not a
+	# bash function (which is what runc and __runc are).
+	setup_runc_cmdline
+
+	first="$(first_cpu)"
+
+	# Running without cpuset should result in an affinity for all CPUs.
+	update_config '.process.args = [ "/bin/sleep", "infinity" ]'
+	update_config 'del(.linux.resources.cpu)'
+	sane_run taskset -c "$first" "${RUNC_CMDLINE[@]}" run -d --console-socket "$CONSOLE_SOCKET" ctr3
+	[ "$status" -eq 0 ]
+	sane_run taskset -c "$first" "${RUNC_CMDLINE[@]}" exec ctr3 grep -F Cpus_allowed_list: /proc/self/status
+	[ "$status" -eq 0 ]
+	[[ "$output" != $'Cpus_allowed_list:\t'"$first" ]]
+	[[ "$output" == $'Cpus_allowed_list:\t'"$INITIAL_CPU_MASK" ]]
+}
+
+@test "runc exec [default CPU affinity should reset to cgroup cpuset]" {
+	[ $EUID -ne 0 ] && requires rootless_cgroup
+	set_cgroups_path
+
+	# We need to use RUNC_CMDLINE since taskset requires a proper binary, not a
+	# bash function (which is what runc and __runc are).
+	setup_runc_cmdline
+
+	first="$(first_cpu)"
+	second="$((first + 1))" # Hacky; might not work in all environments.
+
+	# Running with a cpuset should result in an affinity that matches.
+	update_config '.process.args = [ "/bin/sleep", "infinity" ]'
+	update_config '.linux.resources.cpu = {"mems": "0", "cpus": "'"$first-$second"'"}'
+	sane_run taskset -c "$first" "${RUNC_CMDLINE[@]}" run -d --console-socket "$CONSOLE_SOCKET" ctr
+	[ "$status" -eq 0 ]
+	sane_run taskset -c "$first" "${RUNC_CMDLINE[@]}" exec ctr grep -F Cpus_allowed_list: /proc/self/status
+	[ "$status" -eq 0 ]
+	[[ "$output" != $'Cpus_allowed_list:\t'"$first" ]]
+	# XXX: For some reason, systemd-cgroup leads to us using the all-set
+	#      cpumask rather than the cpuset we configured?
+	[ -v RUNC_USE_SYSTEMD ] || [[ "$output" == $'Cpus_allowed_list:\t'"$first-$second" ]]
+
+	# Stop the container so we can reconfigure it.
+	runc delete -f ctr
+	[ "$status" -eq 0 ]
+
+	# Ditto for a cpuset that has no overlap with the original cpumask.
+	update_config '.linux.resources.cpu = {"mems": "0", "cpus": "'"$second"'"}'
+	sane_run taskset -c "$first" "${RUNC_CMDLINE[@]}" run -d --console-socket "$CONSOLE_SOCKET" ctr
+	[ "$status" -eq 0 ]
+	sane_run taskset -c "$first" "${RUNC_CMDLINE[@]}" exec ctr grep -F Cpus_allowed_list: /proc/self/status
+	[ "$status" -eq 0 ]
+	[[ "$output" != $'Cpus_allowed_list:\t'"$first" ]]
+	# XXX: For some reason, systemd-cgroup leads to us using the all-set
+	#      cpumask rather than the cpuset we configured?
+	[ -v RUNC_USE_SYSTEMD ] || [[ "$output" == $'Cpus_allowed_list:\t'"$second" ]]
+}
@@ -39,22 +39,38 @@ ARCH=$(uname -m)
 # Seccomp agent socket.
 SECCCOMP_AGENT_SOCKET="$BATS_TMPDIR/seccomp-agent.sock"

-# Wrapper for runc.
-function runc() {
-	run __runc "$@"
+# Wrapper around "run" that logs output to make tests easier to debug.
+function sane_run() {
+	local cmd="$1"
+	local cmdname="${CMDNAME:-$(basename "$cmd")}"
+	shift
+
+	run "$cmd" "$@"

 	# Some debug information to make life easier. bats will only print it if the
 	# test failed, in which case the output is useful.
 	# shellcheck disable=SC2154
-	echo "$(basename "$RUNC") $* (status=$status):" >&2
+	echo "$cmdname $* (status=$status)" >&2
 	# shellcheck disable=SC2154
 	echo "$output" >&2
 }

+# Wrapper for runc.
+function runc() {
+	CMDNAME="$(basename "$RUNC")" sane_run __runc "$@"
+}
+
+function setup_runc_cmdline() {
+	RUNC_CMDLINE=("$RUNC")
+	[[ -v RUNC_USE_SYSTEMD ]] && RUNC_CMDLINE+=("--systemd-cgroup")
+	[[ -n "${ROOT:-}" ]] && RUNC_CMDLINE+=("--root" "$ROOT/state")
+	export RUNC_CMDLINE
+}
+
 # Raw wrapper for runc.
 function __runc() {
-	"$RUNC" ${RUNC_USE_SYSTEMD+--systemd-cgroup} \
-		${ROOT:+--root "$ROOT/state"} "$@"
+	setup_runc_cmdline
+	"${RUNC_CMDLINE[@]}" "$@"
 }

 # Wrapper for runc spec.