Merge pull request #3059 from kolyshkin/cgroup-clean

runc exec --cgroup

exec.go

@@ -87,6 +87,10 @@ following will output a list of processes running in the container:
 			Name:  "preserve-fds",
 			Usage: "Pass N additional file descriptors to the container (stdio + $LISTEN_FDS + N in total)",
 		},
+		cli.StringSliceFlag{
+			Name:  "cgroup",
+			Usage: "run the process in an (existing) sub-cgroup(s). Format is [<controller>:]<cgroup>.",
+		},
 	},
 	Action: func(context *cli.Context) error {
 		if err := checkArgs(context, 1, minArgs); err != nil {
@@ -105,6 +109,32 @@ following will output a list of processes running in the container:
 	SkipArgReorder: true,
 }
 
+func getSubCgroupPaths(args []string) (map[string]string, error) {
+	if len(args) == 0 {
+		return nil, nil
+	}
+	paths := make(map[string]string, len(args))
+	for _, c := range args {
+		// Split into controller:path.
+		cs := strings.SplitN(c, ":", 3)
+		if len(cs) > 2 {
+			return nil, fmt.Errorf("invalid --cgroup argument: %s", c)
+		}
+		if len(cs) == 1 { // no controller: prefix
+			if len(args) != 1 {
+				return nil, fmt.Errorf("invalid --cgroup argument: %s (missing <controller>: prefix)", c)
+			}
+			paths[""] = c
+		} else {
+			// There may be a few comma-separated controllers.
+			for _, ctrl := range strings.Split(cs[0], ",") {
+				paths[ctrl] = cs[1]
+			}
+		}
+	}
+	return paths, nil
+}
+
 func execProcess(context *cli.Context) (int, error) {
 	container, err := getContainer(context)
 	if err != nil {
@@ -121,7 +151,6 @@ func execProcess(context *cli.Context) (int, error) {
 	if path == "" && len(context.Args()) == 1 {
 		return -1, errors.New("process args cannot be empty")
 	}
-	detach := context.Bool("detach")
 	state, err := container.State()
 	if err != nil {
 		return -1, err
@@ -132,16 +161,22 @@ func execProcess(context *cli.Context) (int, error) {
 		return -1, err
 	}
 
+	cgPaths, err := getSubCgroupPaths(context.StringSlice("cgroup"))
+	if err != nil {
+		return -1, err
+	}
+
 	r := &runner{
 		enableSubreaper: false,
 		shouldDestroy:   false,
 		container:       container,
 		consoleSocket:   context.String("console-socket"),
-		detach:          detach,
+		detach:          context.Bool("detach"),
 		pidFile:         context.String("pid-file"),
 		action:          CT_ACT_RUN,
 		init:            false,
 		preserveFDs:     context.Int("preserve-fds"),
+		subCgroupPaths:  cgPaths,
 	}
 	return r.run(p)
 }

libcontainer/cgroups/fs/fs.go

@@ -35,6 +35,14 @@ var (
 
 var errSubsystemDoesNotExist = errors.New("cgroup: subsystem does not exist")
 
+func init() {
+	// If using cgroups-hybrid mode then add a "" controller indicating
+	// it should join the cgroups v2.
+	if cgroups.IsCgroup2HybridMode() {
+		subsystems = append(subsystems, &NameGroup{GroupName: "", Join: true})
+	}
+}
+
 type subsystem interface {
 	// Name returns the name of the subsystem.
 	Name() string

libcontainer/cgroups/systemd/v1.go

@@ -143,6 +143,19 @@ func initPaths(c *configs.Cgroup) (map[string]string, error) {
 		}
 		paths[s.Name()] = subsystemPath
 	}
+
+	// If systemd is using cgroups-hybrid mode then add the slice path of
+	// this container to the paths so the following process executed with
+	// "runc exec" joins that cgroup as well.
+	if cgroups.IsCgroup2HybridMode() {
+		// "" means cgroup-hybrid path
+		cgroupsHybridPath, err := getSubsystemPath(slice, unit, "")
+		if err != nil && cgroups.IsNotFound(err) {
+			return nil, err
+		}
+		paths[""] = cgroupsHybridPath
+	}
+
 	return paths, nil
 }
 

libcontainer/cgroups/utils.go

@@ -21,11 +21,14 @@ import (
 const (
 	CgroupProcesses   = "cgroup.procs"
 	unifiedMountpoint = "/sys/fs/cgroup"
+	hybridMountpoint  = "/sys/fs/cgroup/unified"
 )
 
 var (
 	isUnifiedOnce sync.Once
 	isUnified     bool
+	isHybridOnce  sync.Once
+	isHybrid      bool
 )
 
 // IsCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode.
@@ -47,6 +50,24 @@ func IsCgroup2UnifiedMode() bool {
 	return isUnified
 }
 
+// IsCgroup2HybridMode returns whether we are running in cgroup v2 hybrid mode.
+func IsCgroup2HybridMode() bool {
+	isHybridOnce.Do(func() {
+		var st unix.Statfs_t
+		err := unix.Statfs(hybridMountpoint, &st)
+		if err != nil {
+			if os.IsNotExist(err) {
+				// ignore the "not found" error
+				isHybrid = false
+				return
+			}
+			panic(fmt.Sprintf("cannot statfs cgroup root: %s", err))
+		}
+		isHybrid = st.Type == unix.CGROUP2_SUPER_MAGIC
+	})
+	return isHybrid
+}
+
 type Mount struct {
 	Mountpoint string
 	Root       string
@@ -352,7 +373,7 @@ func WriteCgroupProc(dir string, pid int) error {
 
 	file, err := OpenFile(dir, CgroupProcesses, os.O_WRONLY)
 	if err != nil {
-		return fmt.Errorf("failed to write %v to %v: %w", pid, CgroupProcesses, err)
+		return fmt.Errorf("failed to write %v: %w", pid, err)
 	}
 	defer file.Close()
 
@@ -369,7 +390,7 @@ func WriteCgroupProc(dir string, pid int) error {
 			continue
 		}
 
-		return fmt.Errorf("failed to write %v to %v: %w", pid, CgroupProcesses, err)
+		return fmt.Errorf("failed to write %v: %w", pid, err)
 	}
 	return err
 }

libcontainer/cgroups/v1_utils.go

@@ -113,6 +113,11 @@ func FindCgroupMountpoint(cgroupPath, subsystem string) (string, error) {
 		return "", errUnified
 	}
 
+	// If subsystem is empty, we look for the cgroupv2 hybrid path.
+	if len(subsystem) == 0 {
+		return hybridMountpoint, nil
+	}
+
 	// Avoid parsing mountinfo by trying the default path first, if possible.
 	if path := tryDefaultPath(cgroupPath, subsystem); path != "" {
 		return path, nil
@@ -223,6 +228,11 @@ func GetOwnCgroupPath(subsystem string) (string, error) {
 		return "", err
 	}
 
+	// If subsystem is empty, we look for the cgroupv2 hybrid path.
+	if len(subsystem) == 0 {
+		return hybridMountpoint, nil
+	}
+
 	return getCgroupPathHelper(subsystem, cgroup)
 }
 

libcontainer/container_linux.go

@@ -10,6 +10,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"reflect"
 	"strconv"
@@ -561,7 +562,7 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, messageSockP
 	if err != nil {
 		return nil, err
 	}
-	return &setnsProcess{
+	proc := &setnsProcess{
 		cmd:             cmd,
 		cgroupPaths:     state.CgroupPaths,
 		rootlessCgroups: c.config.RootlessCgroups,
@@ -573,7 +574,29 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, messageSockP
 		process:         p,
 		bootstrapData:   data,
 		initProcessPid:  state.InitProcessPid,
-	}, nil
+	}
+	if len(p.SubCgroupPaths) > 0 {
+		if add, ok := p.SubCgroupPaths[""]; ok {
+			// cgroup v1: using the same path for all controllers.
+			// cgroup v2: the only possible way.
+			for k := range proc.cgroupPaths {
+				proc.cgroupPaths[k] = path.Join(proc.cgroupPaths[k], add)
+			}
+			// cgroup v2: do not try to join init process's cgroup
+			// as a fallback (see (*setnsProcess).start).
+			proc.initProcessPid = 0
+		} else {
+			// Per-controller paths.
+			for ctrl, add := range p.SubCgroupPaths {
+				if val, ok := proc.cgroupPaths[ctrl]; ok {
+					proc.cgroupPaths[ctrl] = path.Join(val, add)
+				} else {
+					return nil, fmt.Errorf("unknown controller %s in SubCgroupPaths", ctrl)
+				}
+			}
+		}
+	}
+	return proc, nil
 }
 
 func (c *linuxContainer) newInitConfig(process *Process) *initConfig {

libcontainer/process.go

@@ -80,6 +80,15 @@ type Process struct {
 	ops processOperations
 
 	LogLevel string
+
+	// SubCgroupPaths specifies sub-cgroups to run the process in.
+	// Map keys are controller names, map values are paths (relative to
+	// container's top-level cgroup).
+	//
+	// If empty, the default top-level container's cgroup is used.
+	//
+	// For cgroup v2, the only key allowed is "".
+	SubCgroupPaths map[string]string
 }
 
 // Wait waits for the process to exit.

libcontainer/process_linux.go

@@ -124,12 +124,12 @@ func (p *setnsProcess) start() (retErr error) {
 	if err := p.execSetns(); err != nil {
 		return fmt.Errorf("error executing setns process: %w", err)
 	}
-	if len(p.cgroupPaths) > 0 {
-		if err := cgroups.EnterPid(p.cgroupPaths, p.pid()); err != nil && !p.rootlessCgroups {
-			// On cgroup v2 + nesting + domain controllers, EnterPid may fail with EBUSY.
+	for _, path := range p.cgroupPaths {
+		if err := cgroups.WriteCgroupProc(path, p.pid()); err != nil && !p.rootlessCgroups {
+			// On cgroup v2 + nesting + domain controllers, WriteCgroupProc may fail with EBUSY.
 			// https://github.com/opencontainers/runc/issues/2356#issuecomment-621277643
 			// Try to join the cgroup of InitProcessPid.
-			if cgroups.IsCgroup2UnifiedMode() {
+			if cgroups.IsCgroup2UnifiedMode() && p.initProcessPid != 0 {
 				initProcCgroupFile := fmt.Sprintf("/proc/%d/cgroup", p.initProcessPid)
 				initCg, initCgErr := cgroups.ParseCgroupFile(initProcCgroupFile)
 				if initCgErr == nil {

man/runc-exec.8.md

@@ -59,6 +59,17 @@ multiple times.
 : Pass _N_ additional file descriptors to the container (**stdio** +
 **$LISTEN_FDS** + _N_ in total). Default is **0**.
 
+**--cgroup** _path_ | _controller_[,_controller_...]:_path_
+: Execute a process in a sub-cgroup. If the specified cgroup does not exist, an
+error is returned. Default is empty path, which means to use container's top
+level cgroup.
+: For cgroup v1 only, a particular _controller_ (or multiple comma-separated
+controllers) can be specified, and the option can be used multiple times to set
+different paths for different controllers.
+: Note for cgroup v2, in case the process can't join the top level cgroup,
+**runc exec** fallback is to try joining the cgroup of container's init.
+This fallback can be disabled by using **--cgroup /**.
+
 # EXIT STATUS
 
 Exits with a status of _command_ (unless **-d** is used), or **255** if

tests/integration/cgroups.bats

@@ -282,3 +282,24 @@ function setup() {
 	[ "$status" -eq 0 ]
 	[ "$(wc -l <<<"$output")" -eq 1 ]
 }
+
+@test "runc exec (cgroup v1+hybrid joins correct cgroup)" {
+	requires root cgroups_hybrid
+
+	set_cgroups_path
+
+	runc run --pid-file pid.txt -d --console-socket "$CONSOLE_SOCKET" test_cgroups_group
+	[ "$status" -eq 0 ]
+
+	pid=$(cat pid.txt)
+	run_cgroup=$(tail -1 </proc/"$pid"/cgroup)
+	[[ "$run_cgroup" == *"runc-cgroups-integration-test"* ]]
+
+	runc exec test_cgroups_group cat /proc/self/cgroup
+	[ "$status" -eq 0 ]
+	exec_cgroup=${lines[-1]}
+	[[ $exec_cgroup == *"runc-cgroups-integration-test"* ]]
+
+	# check that the cgroups v2 path is the same for both processes
+	[[ "$run_cgroup" == "$exec_cgroup" ]]
+}

tests/integration/exec.bats

@@ -187,3 +187,114 @@ function check_exec_debug() {
 	[[ "${output}" == *"level=debug"* ]]
 	check_exec_debug "$output"
 }
+
+@test "runc exec --cgroup sub-cgroups [v1]" {
+	requires root cgroups_v1
+
+	set_cgroups_path
+	set_cgroup_mount_writable
+
+	__runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox
+	testcontainer test_busybox running
+
+	# Check we can't join non-existing subcgroup.
+	runc exec --cgroup nonexistent test_busybox cat /proc/self/cgroup
+	[ "$status" -ne 0 ]
+	[[ "$output" == *" adding pid "*"/nonexistent/cgroup.procs: no such file "* ]]
+
+	# Check we can't join non-existing subcgroup (for a particular controller).
+	runc exec --cgroup cpu:nonexistent test_busybox cat /proc/self/cgroup
+	[ "$status" -ne 0 ]
+	[[ "$output" == *" adding pid "*"/nonexistent/cgroup.procs: no such file "* ]]
+
+	# Check we can't specify non-existent controller.
+	runc exec --cgroup whaaat:/ test_busybox true
+	[ "$status" -ne 0 ]
+	[[ "$output" == *"unknown controller "* ]]
+
+	# Check we can join top-level cgroup (implicit).
+	runc exec test_busybox cat /proc/self/cgroup
+	[ "$status" -eq 0 ]
+	! grep -v ":$REL_CGROUPS_PATH\$" <<<"$output"
+
+	# Check we can join top-level cgroup (explicit).
+	runc exec --cgroup / test_busybox cat /proc/self/cgroup
+	[ "$status" -eq 0 ]
+	! grep -v ":$REL_CGROUPS_PATH\$" <<<"$output"
+
+	# Create a few subcgroups.
+	# Note that cpu,cpuacct may be mounted together or separate.
+	runc exec test_busybox sh -euc "mkdir -p /sys/fs/cgroup/memory/submem /sys/fs/cgroup/cpu/subcpu /sys/fs/cgroup/cpuacct/subcpu"
+	[ "$status" -eq 0 ]
+
+	# Check that explicit --cgroup works.
+	runc exec --cgroup memory:submem --cgroup cpu,cpuacct:subcpu test_busybox cat /proc/self/cgroup
+	[ "$status" -eq 0 ]
+	[[ "$output" == *":memory:$REL_CGROUPS_PATH/submem"* ]]
+	[[ "$output" == *":cpu"*":$REL_CGROUPS_PATH/subcpu"* ]]
+}
+
+@test "runc exec --cgroup subcgroup [v2]" {
+	requires root cgroups_v2
+
+	set_cgroups_path
+	set_cgroup_mount_writable
+
+	__runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox
+	testcontainer test_busybox running
+
+	# Check we can't join non-existing subcgroup.
+	runc exec --cgroup nonexistent test_busybox cat /proc/self/cgroup
+	[ "$status" -ne 0 ]
+	[[ "$output" == *" adding pid "*"/nonexistent/cgroup.procs: no such file "* ]]
+
+	# Check we can join top-level cgroup (implicit).
+	runc exec test_busybox grep '^0::/$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Check we can join top-level cgroup (explicit).
+	runc exec --cgroup / test_busybox grep '^0::/$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Now move "init" to a subcgroup, and check it was moved.
+	runc exec test_busybox sh -euc "mkdir /sys/fs/cgroup/foobar \
+		&& echo 1 > /sys/fs/cgroup/foobar/cgroup.procs \
+		&& grep -w foobar /proc/1/cgroup"
+	[ "$status" -eq 0 ]
+
+	# The following part is taken from
+	# @test "runc exec (cgroup v2 + init process in non-root cgroup) succeeds"
+
+	# The init process is now in "/foo", but an exec process can still
+	# join "/" because we haven't enabled any domain controller yet.
+	runc exec test_busybox grep '^0::/$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Turn on a domain controller (memory).
+	runc exec test_busybox sh -euc 'echo $$ > /sys/fs/cgroup/foobar/cgroup.procs; echo +memory > /sys/fs/cgroup/cgroup.subtree_control'
+	[ "$status" -eq 0 ]
+
+	# An exec process can no longer join "/" after turning on a domain
+	# controller.  Check that cgroup v2 fallback to init cgroup works.
+	runc exec test_busybox sh -euc "cat /proc/self/cgroup && grep '^0::/foobar$' /proc/self/cgroup"
+	[ "$status" -eq 0 ]
+
+	# Check that --cgroup / disables the init cgroup fallback.
+	runc exec --cgroup / test_busybox true
+	[ "$status" -ne 0 ]
+	[[ "$output" == *" adding pid "*" to cgroups"*"/cgroup.procs: device or resource busy"* ]]
+
+	# Check that explicit --cgroup foobar works.
+	runc exec --cgroup foobar test_busybox grep '^0::/foobar$' /proc/self/cgroup
+	[ "$status" -eq 0 ]
+
+	# Check all processes is in foobar (this check is redundant).
+	runc exec --cgroup foobar test_busybox sh -euc '! grep -vwH foobar /proc/*/cgroup'
+	[ "$status" -eq 0 ]
+
+	# Add a second subcgroup, check we're in it.
+	runc exec --cgroup foobar test_busybox mkdir /sys/fs/cgroup/second
+	[ "$status" -eq 0 ]
+	runc exec --cgroup second test_busybox grep -w second /proc/self/cgroup
+	[ "$status" -eq 0 ]
+}

tests/integration/helpers.bash

@@ -127,6 +127,9 @@ function init_cgroup_paths() {
 			CGROUP_SUBSYSTEMS+=" freezer"
 		fi
 	else
+		if stat -f -c %t /sys/fs/cgroup/unified | grep -qFw 63677270; then
+			CGROUP_HYBRID=yes
+		fi
 		CGROUP_UNIFIED=no
 		CGROUP_SUBSYSTEMS=$(awk '!/^#/ {print $1}' /proc/cgroups)
 		local g base_path
@@ -410,6 +413,12 @@ function requires() {
 				skip_me=1
 			fi
 			;;
+		cgroups_hybrid)
+			init_cgroup_paths
+			if [ "$CGROUP_HYBRID" != "yes" ]; then
+				skip_me=1
+			fi
+			;;
 		cgroups_*)
 			init_cgroup_paths
 			var=${var#cgroups_}

utils_linux.go

@@ -234,6 +234,7 @@ type runner struct {
 	action          CtAct
 	notifySocket    *notifySocket
 	criuOpts        *libcontainer.CriuOpts
+	subCgroupPaths  map[string]string
 }
 
 func (r *runner) run(config *specs.Process) (int, error) {
@@ -253,6 +254,7 @@ func (r *runner) run(config *specs.Process) (int, error) {
 	process.LogLevel = strconv.Itoa(int(logrus.GetLevel()))
 	// Populate the fields that come from runner.
 	process.Init = r.init
+	process.SubCgroupPaths = r.subCgroupPaths
 	if len(r.listenFDs) > 0 {
 		process.Env = append(process.Env, "LISTEN_FDS="+strconv.Itoa(len(r.listenFDs)), "LISTEN_PID=1")
 		process.ExtraFiles = append(process.ExtraFiles, r.listenFDs...)