runc exec: implement --cgroup

In some setups, multiple cgroups are used inside a container,
and sometime there is a need to execute a process in a particular
sub-cgroup (in case of cgroup v1, for a particular controller).
This is what this commit implements.

Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>

exec.go

@@ -87,6 +87,10 @@ following will output a list of processes running in the container:
 			Name:  "preserve-fds",
 			Usage: "Pass N additional file descriptors to the container (stdio + $LISTEN_FDS + N in total)",
 		},
+		cli.StringSliceFlag{
+			Name:  "cgroup",
+			Usage: "run the process in an (existing) sub-cgroup(s). Format is [<controller>:]<cgroup>.",
+		},
 	},
 	Action: func(context *cli.Context) error {
 		if err := checkArgs(context, 1, minArgs); err != nil {
@@ -105,6 +109,32 @@ following will output a list of processes running in the container:
 	SkipArgReorder: true,
 }
 
+func getSubCgroupPaths(args []string) (map[string]string, error) {
+	if len(args) == 0 {
+		return nil, nil
+	}
+	paths := make(map[string]string, len(args))
+	for _, c := range args {
+		// Split into controller:path.
+		cs := strings.SplitN(c, ":", 3)
+		if len(cs) > 2 {
+			return nil, fmt.Errorf("invalid --cgroup argument: %s", c)
+		}
+		if len(cs) == 1 { // no controller: prefix
+			if len(args) != 1 {
+				return nil, fmt.Errorf("invalid --cgroup argument: %s (missing <controller>: prefix)", c)
+			}
+			paths[""] = c
+		} else {
+			// There may be a few comma-separated controllers.
+			for _, ctrl := range strings.Split(cs[0], ",") {
+				paths[ctrl] = cs[1]
+			}
+		}
+	}
+	return paths, nil
+}
+
 func execProcess(context *cli.Context) (int, error) {
 	container, err := getContainer(context)
 	if err != nil {
@@ -131,6 +161,11 @@ func execProcess(context *cli.Context) (int, error) {
 		return -1, err
 	}
 
+	cgPaths, err := getSubCgroupPaths(context.StringSlice("cgroup"))
+	if err != nil {
+		return -1, err
+	}
+
 	r := &runner{
 		enableSubreaper: false,
 		shouldDestroy:   false,
@@ -141,6 +176,7 @@ func execProcess(context *cli.Context) (int, error) {
 		action:          CT_ACT_RUN,
 		init:            false,
 		preserveFDs:     context.Int("preserve-fds"),
+		subCgroupPaths:  cgPaths,
 	}
 	return r.run(p)
 }

libcontainer/container_linux.go

@@ -10,6 +10,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"reflect"
 	"strconv"
@@ -561,7 +562,7 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, messageSockP
 	if err != nil {
 		return nil, err
 	}
-	return &setnsProcess{
+	proc := &setnsProcess{
 		cmd:             cmd,
 		cgroupPaths:     state.CgroupPaths,
 		rootlessCgroups: c.config.RootlessCgroups,
@@ -573,7 +574,29 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, messageSockP
 		process:         p,
 		bootstrapData:   data,
 		initProcessPid:  state.InitProcessPid,
-	}, nil
+	}
+	if len(p.SubCgroupPaths) > 0 {
+		if add, ok := p.SubCgroupPaths[""]; ok {
+			// cgroup v1: using the same path for all controllers.
+			// cgroup v2: the only possible way.
+			for k := range proc.cgroupPaths {
+				proc.cgroupPaths[k] = path.Join(proc.cgroupPaths[k], add)
+			}
+			// cgroup v2: do not try to join init process's cgroup
+			// as a fallback (see (*setnsProcess).start).
+			proc.initProcessPid = 0
+		} else {
+			// Per-controller paths.
+			for ctrl, add := range p.SubCgroupPaths {
+				if val, ok := proc.cgroupPaths[ctrl]; ok {
+					proc.cgroupPaths[ctrl] = path.Join(val, add)
+				} else {
+					return nil, fmt.Errorf("unknown controller %s in SubCgroupPaths", ctrl)
+				}
+			}
+		}
+	}
+	return proc, nil
 }
 
 func (c *linuxContainer) newInitConfig(process *Process) *initConfig {

libcontainer/process.go

@@ -80,6 +80,15 @@ type Process struct {
 	ops processOperations
 
 	LogLevel string
+
+	// SubCgroupPaths specifies sub-cgroups to run the process in.
+	// Map keys are controller names, map values are paths (relative to
+	// container's top-level cgroup).
+	//
+	// If empty, the default top-level container's cgroup is used.
+	//
+	// For cgroup v2, the only key allowed is "".
+	SubCgroupPaths map[string]string
 }
 
 // Wait waits for the process to exit.

libcontainer/process_linux.go

@@ -129,7 +129,7 @@ func (p *setnsProcess) start() (retErr error) {
 			// On cgroup v2 + nesting + domain controllers, WriteCgroupProc may fail with EBUSY.
 			// https://github.com/opencontainers/runc/issues/2356#issuecomment-621277643
 			// Try to join the cgroup of InitProcessPid.
-			if cgroups.IsCgroup2UnifiedMode() {
+			if cgroups.IsCgroup2UnifiedMode() && p.initProcessPid != 0 {
 				initProcCgroupFile := fmt.Sprintf("/proc/%d/cgroup", p.initProcessPid)
 				initCg, initCgErr := cgroups.ParseCgroupFile(initProcCgroupFile)
 				if initCgErr == nil {

man/runc-exec.8.md

@@ -59,6 +59,17 @@ multiple times.
 : Pass _N_ additional file descriptors to the container (**stdio** +
 **$LISTEN_FDS** + _N_ in total). Default is **0**.
 
+**--cgroup** _path_ | _controller_[,_controller_...]:_path_
+: Execute a process in a sub-cgroup. If the specified cgroup does not exist, an
+error is returned. Default is empty path, which means to use container's top
+level cgroup.
+: For cgroup v1 only, a particular _controller_ (or multiple comma-separated
+controllers) can be specified, and the option can be used multiple times to set
+different paths for different controllers.
+: Note for cgroup v2, in case the process can't join the top level cgroup,
+**runc exec** fallback is to try joining the cgroup of container's init.
+This fallback can be disabled by using **--cgroup /**.
+
 # EXIT STATUS
 
 Exits with a status of _command_ (unless **-d** is used), or **255** if

tests/integration/exec.bats

@@ -187,3 +187,114 @@ function check_exec_debug() {
 	[[ "${output}" == *"level=debug"* ]]
 	check_exec_debug "$output"
 }
+
+@test "runc exec --cgroup sub-cgroups [v1]" {
+	requires root cgroups_v1
+
+	set_cgroups_path
+	set_cgroup_mount_writable
+
+	__runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox
+	testcontainer test_busybox running
+
+	# Check we can't join non-existing subcgroup.
+	runc exec --cgroup nonexistent test_busybox cat /proc/self/cgroup
+	[ "$status" -ne 0 ]
+	[[ "$output" == *" adding pid "*"/nonexistent/cgroup.procs: no such file "* ]]
+
+	# Check we can't join non-existing subcgroup (for a particular controller).
+	runc exec --cgroup cpu:nonexistent test_busybox cat /proc/self/cgroup
+	[ "$status" -ne 0 ]
+	[[ "$output" == *" adding pid "*"/nonexistent/cgroup.procs: no such file "* ]]
+
+	# Check we can't specify non-existent controller.
+	runc exec --cgroup whaaat:/ test_busybox true
+	[ "$status" -ne 0 ]
+	[[ "$output" == *"unknown controller "* ]]
+
+	# Check we can join top-level cgroup (implicit).
+	runc exec test_busybox cat /proc/self/cgroup
+	[ "$status" -eq 0 ]
+	! grep -v ":$REL_CGROUPS_PATH\$" <<<"$output"
+
+	# Check we can join top-level cgroup (explicit).
+	runc exec --cgroup / test_busybox cat /proc/self/cgroup
+	[ "$status" -eq 0 ]
+	! grep -v ":$REL_CGROUPS_PATH\$" <<<"$output"
+
+	# Create a few subcgroups.
+	# Note that cpu,cpuacct may be mounted together or separate.
+	runc exec test_busybox sh -euc "mkdir -p /sys/fs/cgroup/memory/submem /sys/fs/cgroup/cpu/subcpu /sys/fs/cgroup/cpuacct/subcpu"
+	[ "$status" -eq 0 ]
+
+	# Check that explicit --cgroup works.
+	runc exec --cgroup memory:submem --cgroup cpu,cpuacct:subcpu test_busybox cat /proc/self/cgroup
+	[ "$status" -eq 0 ]
+	[[ "$output" == *":memory:$REL_CGROUPS_PATH/submem"* ]]
+	[[ "$output" == *":cpu"*":$REL_CGROUPS_PATH/subcpu"* ]]
+}
+
+@test "runc exec --cgroup subcgroup [v2]" {
+	requires root cgroups_v2
+
+	set_cgroups_path
+	set_cgroup_mount_writable
+
+	__runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox
+	testcontainer test_busybox running
+
+	# Check we can't join non-existing subcgroup.
+	runc exec --cgroup nonexistent test_busybox cat /proc/self/cgroup
+	[ "$status" -ne 0 ]
+	[[ "$output" == *" adding pid "*"/nonexistent/cgroup.procs: no such file "* ]]
+
+	# Check we can join top-level cgroup (implicit).
+	runc exec test_busybox grep '^0::/$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Check we can join top-level cgroup (explicit).
+	runc exec --cgroup / test_busybox grep '^0::/$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Now move "init" to a subcgroup, and check it was moved.
+	runc exec test_busybox sh -euc "mkdir /sys/fs/cgroup/foobar \
+		&& echo 1 > /sys/fs/cgroup/foobar/cgroup.procs \
+		&& grep -w foobar /proc/1/cgroup"
+	[ "$status" -eq 0 ]
+
+	# The following part is taken from
+	# @test "runc exec (cgroup v2 + init process in non-root cgroup) succeeds"
+
+	# The init process is now in "/foo", but an exec process can still
+	# join "/" because we haven't enabled any domain controller yet.
+	runc exec test_busybox grep '^0::/$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Turn on a domain controller (memory).
+	runc exec test_busybox sh -euc 'echo $$ > /sys/fs/cgroup/foobar/cgroup.procs; echo +memory > /sys/fs/cgroup/cgroup.subtree_control'
+	[ "$status" -eq 0 ]
+
+	# An exec process can no longer join "/" after turning on a domain
+	# controller.  Check that cgroup v2 fallback to init cgroup works.
+	runc exec test_busybox sh -euc "cat /proc/self/cgroup && grep '^0::/foobar$' /proc/self/cgroup"
+	[ "$status" -eq 0 ]
+
+	# Check that --cgroup / disables the init cgroup fallback.
+	runc exec --cgroup / test_busybox true
+	[ "$status" -ne 0 ]
+	[[ "$output" == *" adding pid "*" to cgroups"*"/cgroup.procs: device or resource busy"* ]]
+
+	# Check that explicit --cgroup foobar works.
+	runc exec --cgroup foobar test_busybox grep '^0::/foobar$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Check all processes is in foobar (this check is redundant).
+	runc exec --cgroup foobar test_busybox sh -euc '! grep -vwH foobar /proc/*/cgroup'
+	[ "$status" -eq 0 ]
+
+	# Add a second subcgroup, check we're in it.
+	runc exec --cgroup foobar test_busybox mkdir /sys/fs/cgroup/second
+	[ "$status" -eq 0 ]
+	runc exec --cgroup second test_busybox grep -w second /proc/self/cgroup
+	[ "$status" -eq 0 ]
+}

utils_linux.go

@@ -234,6 +234,7 @@ type runner struct {
 	action          CtAct
 	notifySocket    *notifySocket
 	criuOpts        *libcontainer.CriuOpts
+	subCgroupPaths  map[string]string
 }
 
 func (r *runner) run(config *specs.Process) (int, error) {
@@ -253,6 +254,7 @@ func (r *runner) run(config *specs.Process) (int, error) {
 	process.LogLevel = strconv.Itoa(int(logrus.GetLevel()))
 	// Populate the fields that come from runner.
 	process.Init = r.init
+	process.SubCgroupPaths = r.subCgroupPaths
 	if len(r.listenFDs) > 0 {
 		process.Env = append(process.Env, "LISTEN_FDS="+strconv.Itoa(len(r.listenFDs)), "LISTEN_PID=1")
 		process.ExtraFiles = append(process.ExtraFiles, r.listenFDs...)