diff --git a/exec.go b/exec.go index 06b46b408..0d3670551 100644 --- a/exec.go +++ b/exec.go @@ -87,6 +87,10 @@ following will output a list of processes running in the container: Name: "preserve-fds", Usage: "Pass N additional file descriptors to the container (stdio + $LISTEN_FDS + N in total)", }, + cli.StringSliceFlag{ + Name: "cgroup", + Usage: "run the process in an (existing) sub-cgroup(s). Format is [:].", + }, }, Action: func(context *cli.Context) error { if err := checkArgs(context, 1, minArgs); err != nil { @@ -105,6 +109,32 @@ following will output a list of processes running in the container: SkipArgReorder: true, } +func getSubCgroupPaths(args []string) (map[string]string, error) { + if len(args) == 0 { + return nil, nil + } + paths := make(map[string]string, len(args)) + for _, c := range args { + // Split into controller:path. + cs := strings.SplitN(c, ":", 3) + if len(cs) > 2 { + return nil, fmt.Errorf("invalid --cgroup argument: %s", c) + } + if len(cs) == 1 { // no controller: prefix + if len(args) != 1 { + return nil, fmt.Errorf("invalid --cgroup argument: %s (missing : prefix)", c) + } + paths[""] = c + } else { + // There may be a few comma-separated controllers. + for _, ctrl := range strings.Split(cs[0], ",") { + paths[ctrl] = cs[1] + } + } + } + return paths, nil +} + func execProcess(context *cli.Context) (int, error) { container, err := getContainer(context) if err != nil { @@ -131,6 +161,11 @@ func execProcess(context *cli.Context) (int, error) { return -1, err } + cgPaths, err := getSubCgroupPaths(context.StringSlice("cgroup")) + if err != nil { + return -1, err + } + r := &runner{ enableSubreaper: false, shouldDestroy: false, @@ -141,6 +176,7 @@ func execProcess(context *cli.Context) (int, error) { action: CT_ACT_RUN, init: false, preserveFDs: context.Int("preserve-fds"), + subCgroupPaths: cgPaths, } return r.run(p) } diff --git a/libcontainer/container_linux.go b/libcontainer/container_linux.go index de93c6977..df78a950f 100644 --- a/libcontainer/container_linux.go +++ b/libcontainer/container_linux.go @@ -10,6 +10,7 @@ import ( "net" "os" "os/exec" + "path" "path/filepath" "reflect" "strconv" @@ -561,7 +562,7 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, messageSockP if err != nil { return nil, err } - return &setnsProcess{ + proc := &setnsProcess{ cmd: cmd, cgroupPaths: state.CgroupPaths, rootlessCgroups: c.config.RootlessCgroups, @@ -573,7 +574,29 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, messageSockP process: p, bootstrapData: data, initProcessPid: state.InitProcessPid, - }, nil + } + if len(p.SubCgroupPaths) > 0 { + if add, ok := p.SubCgroupPaths[""]; ok { + // cgroup v1: using the same path for all controllers. + // cgroup v2: the only possible way. + for k := range proc.cgroupPaths { + proc.cgroupPaths[k] = path.Join(proc.cgroupPaths[k], add) + } + // cgroup v2: do not try to join init process's cgroup + // as a fallback (see (*setnsProcess).start). + proc.initProcessPid = 0 + } else { + // Per-controller paths. + for ctrl, add := range p.SubCgroupPaths { + if val, ok := proc.cgroupPaths[ctrl]; ok { + proc.cgroupPaths[ctrl] = path.Join(val, add) + } else { + return nil, fmt.Errorf("unknown controller %s in SubCgroupPaths", ctrl) + } + } + } + } + return proc, nil } func (c *linuxContainer) newInitConfig(process *Process) *initConfig { diff --git a/libcontainer/process.go b/libcontainer/process.go index 7cd180389..8a5d340da 100644 --- a/libcontainer/process.go +++ b/libcontainer/process.go @@ -80,6 +80,15 @@ type Process struct { ops processOperations LogLevel string + + // SubCgroupPaths specifies sub-cgroups to run the process in. + // Map keys are controller names, map values are paths (relative to + // container's top-level cgroup). + // + // If empty, the default top-level container's cgroup is used. + // + // For cgroup v2, the only key allowed is "". + SubCgroupPaths map[string]string } // Wait waits for the process to exit. diff --git a/libcontainer/process_linux.go b/libcontainer/process_linux.go index 696ddc5d5..ed361028a 100644 --- a/libcontainer/process_linux.go +++ b/libcontainer/process_linux.go @@ -129,7 +129,7 @@ func (p *setnsProcess) start() (retErr error) { // On cgroup v2 + nesting + domain controllers, WriteCgroupProc may fail with EBUSY. // https://github.com/opencontainers/runc/issues/2356#issuecomment-621277643 // Try to join the cgroup of InitProcessPid. - if cgroups.IsCgroup2UnifiedMode() { + if cgroups.IsCgroup2UnifiedMode() && p.initProcessPid != 0 { initProcCgroupFile := fmt.Sprintf("/proc/%d/cgroup", p.initProcessPid) initCg, initCgErr := cgroups.ParseCgroupFile(initProcCgroupFile) if initCgErr == nil { diff --git a/man/runc-exec.8.md b/man/runc-exec.8.md index 49f9cab53..24e9ff3c9 100644 --- a/man/runc-exec.8.md +++ b/man/runc-exec.8.md @@ -59,6 +59,17 @@ multiple times. : Pass _N_ additional file descriptors to the container (**stdio** + **$LISTEN_FDS** + _N_ in total). Default is **0**. +**--cgroup** _path_ | _controller_[,_controller_...]:_path_ +: Execute a process in a sub-cgroup. If the specified cgroup does not exist, an +error is returned. Default is empty path, which means to use container's top +level cgroup. +: For cgroup v1 only, a particular _controller_ (or multiple comma-separated +controllers) can be specified, and the option can be used multiple times to set +different paths for different controllers. +: Note for cgroup v2, in case the process can't join the top level cgroup, +**runc exec** fallback is to try joining the cgroup of container's init. +This fallback can be disabled by using **--cgroup /**. + # EXIT STATUS Exits with a status of _command_ (unless **-d** is used), or **255** if diff --git a/tests/integration/exec.bats b/tests/integration/exec.bats index a49112fa0..24444d1ec 100644 --- a/tests/integration/exec.bats +++ b/tests/integration/exec.bats @@ -187,3 +187,114 @@ function check_exec_debug() { [[ "${output}" == *"level=debug"* ]] check_exec_debug "$output" } + +@test "runc exec --cgroup sub-cgroups [v1]" { + requires root cgroups_v1 + + set_cgroups_path + set_cgroup_mount_writable + + __runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox + testcontainer test_busybox running + + # Check we can't join non-existing subcgroup. + runc exec --cgroup nonexistent test_busybox cat /proc/self/cgroup + [ "$status" -ne 0 ] + [[ "$output" == *" adding pid "*"/nonexistent/cgroup.procs: no such file "* ]] + + # Check we can't join non-existing subcgroup (for a particular controller). + runc exec --cgroup cpu:nonexistent test_busybox cat /proc/self/cgroup + [ "$status" -ne 0 ] + [[ "$output" == *" adding pid "*"/nonexistent/cgroup.procs: no such file "* ]] + + # Check we can't specify non-existent controller. + runc exec --cgroup whaaat:/ test_busybox true + [ "$status" -ne 0 ] + [[ "$output" == *"unknown controller "* ]] + + # Check we can join top-level cgroup (implicit). + runc exec test_busybox cat /proc/self/cgroup + [ "$status" -eq 0 ] + ! grep -v ":$REL_CGROUPS_PATH\$" <<<"$output" + + # Check we can join top-level cgroup (explicit). + runc exec --cgroup / test_busybox cat /proc/self/cgroup + [ "$status" -eq 0 ] + ! grep -v ":$REL_CGROUPS_PATH\$" <<<"$output" + + # Create a few subcgroups. + # Note that cpu,cpuacct may be mounted together or separate. + runc exec test_busybox sh -euc "mkdir -p /sys/fs/cgroup/memory/submem /sys/fs/cgroup/cpu/subcpu /sys/fs/cgroup/cpuacct/subcpu" + [ "$status" -eq 0 ] + + # Check that explicit --cgroup works. + runc exec --cgroup memory:submem --cgroup cpu,cpuacct:subcpu test_busybox cat /proc/self/cgroup + [ "$status" -eq 0 ] + [[ "$output" == *":memory:$REL_CGROUPS_PATH/submem"* ]] + [[ "$output" == *":cpu"*":$REL_CGROUPS_PATH/subcpu"* ]] +} + +@test "runc exec --cgroup subcgroup [v2]" { + requires root cgroups_v2 + + set_cgroups_path + set_cgroup_mount_writable + + __runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox + testcontainer test_busybox running + + # Check we can't join non-existing subcgroup. + runc exec --cgroup nonexistent test_busybox cat /proc/self/cgroup + [ "$status" -ne 0 ] + [[ "$output" == *" adding pid "*"/nonexistent/cgroup.procs: no such file "* ]] + + # Check we can join top-level cgroup (implicit). + runc exec test_busybox grep '^0::/$' /proc/self/cgroup + [ "$status" -eq 0 ] + + # Check we can join top-level cgroup (explicit). + runc exec --cgroup / test_busybox grep '^0::/$' /proc/self/cgroup + [ "$status" -eq 0 ] + + # Now move "init" to a subcgroup, and check it was moved. + runc exec test_busybox sh -euc "mkdir /sys/fs/cgroup/foobar \ + && echo 1 > /sys/fs/cgroup/foobar/cgroup.procs \ + && grep -w foobar /proc/1/cgroup" + [ "$status" -eq 0 ] + + # The following part is taken from + # @test "runc exec (cgroup v2 + init process in non-root cgroup) succeeds" + + # The init process is now in "/foo", but an exec process can still + # join "/" because we haven't enabled any domain controller yet. + runc exec test_busybox grep '^0::/$' /proc/self/cgroup + [ "$status" -eq 0 ] + + # Turn on a domain controller (memory). + runc exec test_busybox sh -euc 'echo $$ > /sys/fs/cgroup/foobar/cgroup.procs; echo +memory > /sys/fs/cgroup/cgroup.subtree_control' + [ "$status" -eq 0 ] + + # An exec process can no longer join "/" after turning on a domain + # controller. Check that cgroup v2 fallback to init cgroup works. + runc exec test_busybox sh -euc "cat /proc/self/cgroup && grep '^0::/foobar$' /proc/self/cgroup" + [ "$status" -eq 0 ] + + # Check that --cgroup / disables the init cgroup fallback. + runc exec --cgroup / test_busybox true + [ "$status" -ne 0 ] + [[ "$output" == *" adding pid "*" to cgroups"*"/cgroup.procs: device or resource busy"* ]] + + # Check that explicit --cgroup foobar works. + runc exec --cgroup foobar test_busybox grep '^0::/foobar$' /proc/self/cgroup + [ "$status" -eq 0 ] + + # Check all processes is in foobar (this check is redundant). + runc exec --cgroup foobar test_busybox sh -euc '! grep -vwH foobar /proc/*/cgroup' + [ "$status" -eq 0 ] + + # Add a second subcgroup, check we're in it. + runc exec --cgroup foobar test_busybox mkdir /sys/fs/cgroup/second + [ "$status" -eq 0 ] + runc exec --cgroup second test_busybox grep -w second /proc/self/cgroup + [ "$status" -eq 0 ] +} diff --git a/utils_linux.go b/utils_linux.go index 78ad6e242..2e626425f 100644 --- a/utils_linux.go +++ b/utils_linux.go @@ -234,6 +234,7 @@ type runner struct { action CtAct notifySocket *notifySocket criuOpts *libcontainer.CriuOpts + subCgroupPaths map[string]string } func (r *runner) run(config *specs.Process) (int, error) { @@ -253,6 +254,7 @@ func (r *runner) run(config *specs.Process) (int, error) { process.LogLevel = strconv.Itoa(int(logrus.GetLevel())) // Populate the fields that come from runner. process.Init = r.init + process.SubCgroupPaths = r.subCgroupPaths if len(r.listenFDs) > 0 { process.Env = append(process.Env, "LISTEN_FDS="+strconv.Itoa(len(r.listenFDs)), "LISTEN_PID=1") process.ExtraFiles = append(process.ExtraFiles, r.listenFDs...)