diff --git a/exec.go b/exec.go index 66ee3653c..0d3670551 100644 --- a/exec.go +++ b/exec.go @@ -87,6 +87,10 @@ following will output a list of processes running in the container: Name: "preserve-fds", Usage: "Pass N additional file descriptors to the container (stdio + $LISTEN_FDS + N in total)", }, + cli.StringSliceFlag{ + Name: "cgroup", + Usage: "run the process in an (existing) sub-cgroup(s). Format is [:].", + }, }, Action: func(context *cli.Context) error { if err := checkArgs(context, 1, minArgs); err != nil { @@ -105,6 +109,32 @@ following will output a list of processes running in the container: SkipArgReorder: true, } +func getSubCgroupPaths(args []string) (map[string]string, error) { + if len(args) == 0 { + return nil, nil + } + paths := make(map[string]string, len(args)) + for _, c := range args { + // Split into controller:path. + cs := strings.SplitN(c, ":", 3) + if len(cs) > 2 { + return nil, fmt.Errorf("invalid --cgroup argument: %s", c) + } + if len(cs) == 1 { // no controller: prefix + if len(args) != 1 { + return nil, fmt.Errorf("invalid --cgroup argument: %s (missing : prefix)", c) + } + paths[""] = c + } else { + // There may be a few comma-separated controllers. + for _, ctrl := range strings.Split(cs[0], ",") { + paths[ctrl] = cs[1] + } + } + } + return paths, nil +} + func execProcess(context *cli.Context) (int, error) { container, err := getContainer(context) if err != nil { @@ -121,7 +151,6 @@ func execProcess(context *cli.Context) (int, error) { if path == "" && len(context.Args()) == 1 { return -1, errors.New("process args cannot be empty") } - detach := context.Bool("detach") state, err := container.State() if err != nil { return -1, err @@ -132,16 +161,22 @@ func execProcess(context *cli.Context) (int, error) { return -1, err } + cgPaths, err := getSubCgroupPaths(context.StringSlice("cgroup")) + if err != nil { + return -1, err + } + r := &runner{ enableSubreaper: false, shouldDestroy: false, container: container, consoleSocket: context.String("console-socket"), - detach: detach, + detach: context.Bool("detach"), pidFile: context.String("pid-file"), action: CT_ACT_RUN, init: false, preserveFDs: context.Int("preserve-fds"), + subCgroupPaths: cgPaths, } return r.run(p) } diff --git a/libcontainer/cgroups/fs/fs.go b/libcontainer/cgroups/fs/fs.go index a35f628a8..b574d3a05 100644 --- a/libcontainer/cgroups/fs/fs.go +++ b/libcontainer/cgroups/fs/fs.go @@ -35,6 +35,14 @@ var ( var errSubsystemDoesNotExist = errors.New("cgroup: subsystem does not exist") +func init() { + // If using cgroups-hybrid mode then add a "" controller indicating + // it should join the cgroups v2. + if cgroups.IsCgroup2HybridMode() { + subsystems = append(subsystems, &NameGroup{GroupName: "", Join: true}) + } +} + type subsystem interface { // Name returns the name of the subsystem. Name() string diff --git a/libcontainer/cgroups/systemd/v1.go b/libcontainer/cgroups/systemd/v1.go index 1e54cc42c..a74a05a5c 100644 --- a/libcontainer/cgroups/systemd/v1.go +++ b/libcontainer/cgroups/systemd/v1.go @@ -143,6 +143,19 @@ func initPaths(c *configs.Cgroup) (map[string]string, error) { } paths[s.Name()] = subsystemPath } + + // If systemd is using cgroups-hybrid mode then add the slice path of + // this container to the paths so the following process executed with + // "runc exec" joins that cgroup as well. + if cgroups.IsCgroup2HybridMode() { + // "" means cgroup-hybrid path + cgroupsHybridPath, err := getSubsystemPath(slice, unit, "") + if err != nil && cgroups.IsNotFound(err) { + return nil, err + } + paths[""] = cgroupsHybridPath + } + return paths, nil } diff --git a/libcontainer/cgroups/utils.go b/libcontainer/cgroups/utils.go index ec4e70198..3bab0e75a 100644 --- a/libcontainer/cgroups/utils.go +++ b/libcontainer/cgroups/utils.go @@ -21,11 +21,14 @@ import ( const ( CgroupProcesses = "cgroup.procs" unifiedMountpoint = "/sys/fs/cgroup" + hybridMountpoint = "/sys/fs/cgroup/unified" ) var ( isUnifiedOnce sync.Once isUnified bool + isHybridOnce sync.Once + isHybrid bool ) // IsCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode. @@ -47,6 +50,24 @@ func IsCgroup2UnifiedMode() bool { return isUnified } +// IsCgroup2HybridMode returns whether we are running in cgroup v2 hybrid mode. +func IsCgroup2HybridMode() bool { + isHybridOnce.Do(func() { + var st unix.Statfs_t + err := unix.Statfs(hybridMountpoint, &st) + if err != nil { + if os.IsNotExist(err) { + // ignore the "not found" error + isHybrid = false + return + } + panic(fmt.Sprintf("cannot statfs cgroup root: %s", err)) + } + isHybrid = st.Type == unix.CGROUP2_SUPER_MAGIC + }) + return isHybrid +} + type Mount struct { Mountpoint string Root string @@ -352,7 +373,7 @@ func WriteCgroupProc(dir string, pid int) error { file, err := OpenFile(dir, CgroupProcesses, os.O_WRONLY) if err != nil { - return fmt.Errorf("failed to write %v to %v: %w", pid, CgroupProcesses, err) + return fmt.Errorf("failed to write %v: %w", pid, err) } defer file.Close() @@ -369,7 +390,7 @@ func WriteCgroupProc(dir string, pid int) error { continue } - return fmt.Errorf("failed to write %v to %v: %w", pid, CgroupProcesses, err) + return fmt.Errorf("failed to write %v: %w", pid, err) } return err } diff --git a/libcontainer/cgroups/v1_utils.go b/libcontainer/cgroups/v1_utils.go index d850a36b3..47c75f22b 100644 --- a/libcontainer/cgroups/v1_utils.go +++ b/libcontainer/cgroups/v1_utils.go @@ -113,6 +113,11 @@ func FindCgroupMountpoint(cgroupPath, subsystem string) (string, error) { return "", errUnified } + // If subsystem is empty, we look for the cgroupv2 hybrid path. + if len(subsystem) == 0 { + return hybridMountpoint, nil + } + // Avoid parsing mountinfo by trying the default path first, if possible. if path := tryDefaultPath(cgroupPath, subsystem); path != "" { return path, nil @@ -223,6 +228,11 @@ func GetOwnCgroupPath(subsystem string) (string, error) { return "", err } + // If subsystem is empty, we look for the cgroupv2 hybrid path. + if len(subsystem) == 0 { + return hybridMountpoint, nil + } + return getCgroupPathHelper(subsystem, cgroup) } diff --git a/libcontainer/container_linux.go b/libcontainer/container_linux.go index de93c6977..df78a950f 100644 --- a/libcontainer/container_linux.go +++ b/libcontainer/container_linux.go @@ -10,6 +10,7 @@ import ( "net" "os" "os/exec" + "path" "path/filepath" "reflect" "strconv" @@ -561,7 +562,7 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, messageSockP if err != nil { return nil, err } - return &setnsProcess{ + proc := &setnsProcess{ cmd: cmd, cgroupPaths: state.CgroupPaths, rootlessCgroups: c.config.RootlessCgroups, @@ -573,7 +574,29 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, messageSockP process: p, bootstrapData: data, initProcessPid: state.InitProcessPid, - }, nil + } + if len(p.SubCgroupPaths) > 0 { + if add, ok := p.SubCgroupPaths[""]; ok { + // cgroup v1: using the same path for all controllers. + // cgroup v2: the only possible way. + for k := range proc.cgroupPaths { + proc.cgroupPaths[k] = path.Join(proc.cgroupPaths[k], add) + } + // cgroup v2: do not try to join init process's cgroup + // as a fallback (see (*setnsProcess).start). + proc.initProcessPid = 0 + } else { + // Per-controller paths. + for ctrl, add := range p.SubCgroupPaths { + if val, ok := proc.cgroupPaths[ctrl]; ok { + proc.cgroupPaths[ctrl] = path.Join(val, add) + } else { + return nil, fmt.Errorf("unknown controller %s in SubCgroupPaths", ctrl) + } + } + } + } + return proc, nil } func (c *linuxContainer) newInitConfig(process *Process) *initConfig { diff --git a/libcontainer/process.go b/libcontainer/process.go index 7cd180389..8a5d340da 100644 --- a/libcontainer/process.go +++ b/libcontainer/process.go @@ -80,6 +80,15 @@ type Process struct { ops processOperations LogLevel string + + // SubCgroupPaths specifies sub-cgroups to run the process in. + // Map keys are controller names, map values are paths (relative to + // container's top-level cgroup). + // + // If empty, the default top-level container's cgroup is used. + // + // For cgroup v2, the only key allowed is "". + SubCgroupPaths map[string]string } // Wait waits for the process to exit. diff --git a/libcontainer/process_linux.go b/libcontainer/process_linux.go index 13812e731..ed361028a 100644 --- a/libcontainer/process_linux.go +++ b/libcontainer/process_linux.go @@ -124,12 +124,12 @@ func (p *setnsProcess) start() (retErr error) { if err := p.execSetns(); err != nil { return fmt.Errorf("error executing setns process: %w", err) } - if len(p.cgroupPaths) > 0 { - if err := cgroups.EnterPid(p.cgroupPaths, p.pid()); err != nil && !p.rootlessCgroups { - // On cgroup v2 + nesting + domain controllers, EnterPid may fail with EBUSY. + for _, path := range p.cgroupPaths { + if err := cgroups.WriteCgroupProc(path, p.pid()); err != nil && !p.rootlessCgroups { + // On cgroup v2 + nesting + domain controllers, WriteCgroupProc may fail with EBUSY. // https://github.com/opencontainers/runc/issues/2356#issuecomment-621277643 // Try to join the cgroup of InitProcessPid. - if cgroups.IsCgroup2UnifiedMode() { + if cgroups.IsCgroup2UnifiedMode() && p.initProcessPid != 0 { initProcCgroupFile := fmt.Sprintf("/proc/%d/cgroup", p.initProcessPid) initCg, initCgErr := cgroups.ParseCgroupFile(initProcCgroupFile) if initCgErr == nil { diff --git a/man/runc-exec.8.md b/man/runc-exec.8.md index 49f9cab53..24e9ff3c9 100644 --- a/man/runc-exec.8.md +++ b/man/runc-exec.8.md @@ -59,6 +59,17 @@ multiple times. : Pass _N_ additional file descriptors to the container (**stdio** + **$LISTEN_FDS** + _N_ in total). Default is **0**. +**--cgroup** _path_ | _controller_[,_controller_...]:_path_ +: Execute a process in a sub-cgroup. If the specified cgroup does not exist, an +error is returned. Default is empty path, which means to use container's top +level cgroup. +: For cgroup v1 only, a particular _controller_ (or multiple comma-separated +controllers) can be specified, and the option can be used multiple times to set +different paths for different controllers. +: Note for cgroup v2, in case the process can't join the top level cgroup, +**runc exec** fallback is to try joining the cgroup of container's init. +This fallback can be disabled by using **--cgroup /**. + # EXIT STATUS Exits with a status of _command_ (unless **-d** is used), or **255** if diff --git a/tests/integration/cgroups.bats b/tests/integration/cgroups.bats index ee95a01f6..57e81b15c 100644 --- a/tests/integration/cgroups.bats +++ b/tests/integration/cgroups.bats @@ -282,3 +282,24 @@ function setup() { [ "$status" -eq 0 ] [ "$(wc -l <<<"$output")" -eq 1 ] } + +@test "runc exec (cgroup v1+hybrid joins correct cgroup)" { + requires root cgroups_hybrid + + set_cgroups_path + + runc run --pid-file pid.txt -d --console-socket "$CONSOLE_SOCKET" test_cgroups_group + [ "$status" -eq 0 ] + + pid=$(cat pid.txt) + run_cgroup=$(tail -1 /sys/fs/cgroup/foobar/cgroup.procs \ + && grep -w foobar /proc/1/cgroup" + [ "$status" -eq 0 ] + + # The following part is taken from + # @test "runc exec (cgroup v2 + init process in non-root cgroup) succeeds" + + # The init process is now in "/foo", but an exec process can still + # join "/" because we haven't enabled any domain controller yet. + runc exec test_busybox grep '^0::/$' /proc/self/cgroup + [ "$status" -eq 0 ] + + # Turn on a domain controller (memory). + runc exec test_busybox sh -euc 'echo $$ > /sys/fs/cgroup/foobar/cgroup.procs; echo +memory > /sys/fs/cgroup/cgroup.subtree_control' + [ "$status" -eq 0 ] + + # An exec process can no longer join "/" after turning on a domain + # controller. Check that cgroup v2 fallback to init cgroup works. + runc exec test_busybox sh -euc "cat /proc/self/cgroup && grep '^0::/foobar$' /proc/self/cgroup" + [ "$status" -eq 0 ] + + # Check that --cgroup / disables the init cgroup fallback. + runc exec --cgroup / test_busybox true + [ "$status" -ne 0 ] + [[ "$output" == *" adding pid "*" to cgroups"*"/cgroup.procs: device or resource busy"* ]] + + # Check that explicit --cgroup foobar works. + runc exec --cgroup foobar test_busybox grep '^0::/foobar$' /proc/self/cgroup + [ "$status" -eq 0 ] + + # Check all processes is in foobar (this check is redundant). + runc exec --cgroup foobar test_busybox sh -euc '! grep -vwH foobar /proc/*/cgroup' + [ "$status" -eq 0 ] + + # Add a second subcgroup, check we're in it. + runc exec --cgroup foobar test_busybox mkdir /sys/fs/cgroup/second + [ "$status" -eq 0 ] + runc exec --cgroup second test_busybox grep -w second /proc/self/cgroup + [ "$status" -eq 0 ] +} diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash index 4b48e6add..1e0d1d131 100644 --- a/tests/integration/helpers.bash +++ b/tests/integration/helpers.bash @@ -127,6 +127,9 @@ function init_cgroup_paths() { CGROUP_SUBSYSTEMS+=" freezer" fi else + if stat -f -c %t /sys/fs/cgroup/unified | grep -qFw 63677270; then + CGROUP_HYBRID=yes + fi CGROUP_UNIFIED=no CGROUP_SUBSYSTEMS=$(awk '!/^#/ {print $1}' /proc/cgroups) local g base_path @@ -410,6 +413,12 @@ function requires() { skip_me=1 fi ;; + cgroups_hybrid) + init_cgroup_paths + if [ "$CGROUP_HYBRID" != "yes" ]; then + skip_me=1 + fi + ;; cgroups_*) init_cgroup_paths var=${var#cgroups_} diff --git a/utils_linux.go b/utils_linux.go index 78ad6e242..2e626425f 100644 --- a/utils_linux.go +++ b/utils_linux.go @@ -234,6 +234,7 @@ type runner struct { action CtAct notifySocket *notifySocket criuOpts *libcontainer.CriuOpts + subCgroupPaths map[string]string } func (r *runner) run(config *specs.Process) (int, error) { @@ -253,6 +254,7 @@ func (r *runner) run(config *specs.Process) (int, error) { process.LogLevel = strconv.Itoa(int(logrus.GetLevel())) // Populate the fields that come from runner. process.Init = r.init + process.SubCgroupPaths = r.subCgroupPaths if len(r.listenFDs) > 0 { process.Env = append(process.Env, "LISTEN_FDS="+strconv.Itoa(len(r.listenFDs)), "LISTEN_PID=1") process.ExtraFiles = append(process.ExtraFiles, r.listenFDs...)