libct: reuse tmpfs for directory masks

Kubernetes may add one sysfs thermal_throttle entry per CPU to
maskedPaths. On large Intel systems this can produce many directory
masks for a single container. runc currently handles each directory
mask with a separate read-only tmpfs mount, and therefore a separate
tmpfs superblock.

On Linux 4.18/RHEL 8 kernels, creating and tearing down many tmpfs
superblocks can contend on the global shrinker_rwsem when containers
start or stop concurrently.

Use one read-only tmpfs for directory masks and bind-mount it over the
remaining directory targets. The first non-procfs-fd directory mount is
reopened through the container root fd before it is reused. File masks
still bind /dev/null, and procfs fd targets keep the existing
one-tmpfs-per-target behaviour because they are fd aliases rather than
stable rootfs paths.

If the bind-mount of the shared source fails (e.g. due to kernel
restrictions), fall back to individual tmpfs mounts for all remaining
directories. Tmpfs mounts use nr_blocks=1,nr_inodes=1 to minimise
kernel resource usage.

The bind mounts do not create additional tmpfs superblocks. They also
retain the read-only mount flag inherited from the source vfsmount, so
the masking semantics remain unchanged.

xref: kubernetes/kubernetes#138512
xref: kubernetes/kubernetes#138388
xref: kubernetes/kubernetes#131018

Co-authored-by: Davanum Srinivas <davanum@gmail.com>
Refactored-by: lifubang <lifubang@acmcoder.com>
Signed-off-by: lifubang <lifubang@acmcoder.com>

libcontainer/rootfs_linux.go

@@ -419,7 +419,7 @@ func mountCgroupV2(m mountEntry, c *mountConfig) error {
 		// Mask `/sys/fs/cgroup` to ensure it is read-only, even when `/sys` is mounted
 		// with `rbind,ro` (`runc spec --rootless` produces `rbind,ro` for `/sys`).
 		err = utils.WithProcfdFile(m.dstFile, func(procfd string) error {
-			return maskPaths([]string{procfd}, c.label)
+			return maskPaths(c.root, []string{procfd}, c.label)
 		})
 	}
 	return err
@@ -1333,7 +1333,7 @@ func verifyDevNull(f *os.File) error {
 // mounts ( proc/kcore ).
 // For files, maskPath bind mounts /dev/null over the top of the specified path.
 // For directories, maskPath mounts read-only tmpfs over the top of the specified path.
-func maskPaths(paths []string, mountLabel string) error {
+func maskPaths(rootFd *os.File, paths []string, mountLabel string) error {
 	devNull, err := os.OpenFile("/dev/null", unix.O_PATH, 0)
 	if err != nil {
 		return fmt.Errorf("can't mask paths: %w", err)
@@ -1346,6 +1346,16 @@ func maskPaths(paths []string, mountLabel string) error {
 	procSelfFd, closer := utils.ProcThreadSelf("fd/")
 	defer closer()
 
+	var (
+		sharedMaskFile *os.File
+		sharedMaskSrc  *mountSource
+		bindFailed     bool
+	)
+	defer func() {
+		if sharedMaskFile != nil {
+			_ = sharedMaskFile.Close()
+		}
+	}()
 	maskedPaths := make(map[string]struct{})
 	for _, path := range paths {
 		// Open the target path; skip if it doesn't exist.
@@ -1373,7 +1383,31 @@ func maskPaths(paths []string, mountLabel string) error {
 		if st.IsDir() {
 			// Destination is a directory: bind mount a ro tmpfs over it.
 			dstType = "dir"
-			err = mount("tmpfs", path, "tmpfs", unix.MS_RDONLY, label.FormatMountLabel("nr_blocks=1,nr_inodes=1", mountLabel))
+			if !bindFailed && sharedMaskSrc != nil {
+				dstFd := filepath.Join(procSelfFd, strconv.Itoa(int(dstFh.Fd())))
+				err = mountViaFds("", sharedMaskSrc, path, dstFd, "", unix.MS_BIND, "")
+				if err != nil {
+					// A bind-mount inherits MNT_READONLY from the source vfsmount,
+					// but if it fails fall back to individual tmpfs mounts.
+					bindFailed = true
+					logrus.WithError(err).Warn("maskPaths: shared tmpfs bind-mount failed, falling back to per-directory tmpfs")
+				}
+			}
+			if bindFailed || sharedMaskSrc == nil {
+				err = mount("tmpfs", path, "tmpfs", unix.MS_RDONLY, label.FormatMountLabel("nr_blocks=1,nr_inodes=1", mountLabel))
+				if err == nil && !bindFailed && sharedMaskSrc == nil {
+					// Establish this mount as the reusable shared source. reopenAfterMount
+					// resolves the underlying inode via procfs and re-opens it through
+					// rootFd, so the resulting fd is anchored to the real path inside the
+					// container rootfs even if path was a /proc/self/fd/N alias.
+					reopened, err := reopenAfterMount(rootFd, dstFh, unix.O_PATH|unix.O_CLOEXEC)
+					if err != nil {
+						return fmt.Errorf("can't reopen shared directory mask: %w", err)
+					}
+					sharedMaskFile = reopened
+					sharedMaskSrc = &mountSource{Type: mountSourcePlain, file: sharedMaskFile}
+				}
+			}
 		} else {
 			// Destination is a file: mount it to /dev/null.
 			dstType = "path"

libcontainer/standard_init_linux.go

@@ -142,8 +142,16 @@ func (l *linuxStandardInit) Init() error {
 		}
 	}
 
-	if err := maskPaths(l.config.Config.MaskPaths, l.config.Config.MountLabel); err != nil {
+	if len(l.config.Config.MaskPaths) > 0 {
-		return err
+		rootFd, err := os.OpenFile("/", unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_PATH, 0)
+		if err != nil {
+			return fmt.Errorf("open rootfs handle for masked paths: %w", err)
+		}
+		err = maskPaths(rootFd, l.config.Config.MaskPaths, l.config.Config.MountLabel)
+		rootFd.Close()
+		if err != nil {
+			return err
+		}
 	}
 	pdeath, err := system.GetParentDeathSignal()
 	if err != nil {