From 28cb3218873f7b073fba69c74663c9c1ffdfbe6b Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Wed, 18 Mar 2026 17:21:55 -0700 Subject: [PATCH] Pre-open container root directory A lot of filesystem-related stuff happens inside the container root directory, and we have used its name before. It makes sense to pre-open it and use a *os.File handle instead. Function names in internal/pathrs are kept as is for simplicity (and it is an internal package), but they now accept root as *os.File. Signed-off-by: Kir Kolyshkin --- internal/pathrs/mkdirall.go | 4 +- internal/pathrs/mkdirall_pathrslite.go | 15 ++----- internal/pathrs/root_pathrslite.go | 10 ++--- libcontainer/criu_linux.go | 10 ++++- libcontainer/rootfs_linux.go | 55 +++++++++++++++++--------- 5 files changed, 56 insertions(+), 38 deletions(-) diff --git a/internal/pathrs/mkdirall.go b/internal/pathrs/mkdirall.go index 3a896f484..81c9a022c 100644 --- a/internal/pathrs/mkdirall.go +++ b/internal/pathrs/mkdirall.go @@ -31,12 +31,12 @@ import ( // Callers need to be very careful operating on the trailing path, as trivial // mistakes like following symlinks can cause security bugs. Most people // should probably just use [MkdirAllInRoot] or [CreateInRoot]. -func MkdirAllParentInRoot(root, unsafePath string, mode os.FileMode) (*os.File, string, error) { +func MkdirAllParentInRoot(root *os.File, unsafePath string, mode os.FileMode) (*os.File, string, error) { // MkdirAllInRoot also does hallucinateUnsafePath, but we need to do it // here first because when we split unsafePath into (dir, file) components // we want to be doing so with the hallucinated path (so that trailing // dangling symlinks are treated correctly). - unsafePath, err := hallucinateUnsafePath(root, unsafePath) + unsafePath, err := hallucinateUnsafePath(root.Name(), unsafePath) if err != nil { return nil, "", fmt.Errorf("failed to construct hallucinated target path: %w", err) } diff --git a/internal/pathrs/mkdirall_pathrslite.go b/internal/pathrs/mkdirall_pathrslite.go index c2578e051..0a6f25ebd 100644 --- a/internal/pathrs/mkdirall_pathrslite.go +++ b/internal/pathrs/mkdirall_pathrslite.go @@ -24,12 +24,11 @@ import ( "github.com/cyphar/filepath-securejoin/pathrs-lite" "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" ) // MkdirAllInRoot attempts to make // -// path, _ := securejoin.SecureJoin(root, unsafePath) +// path, _ := securejoin.SecureJoin(root.Name(), unsafePath) // os.MkdirAll(path, mode) // os.Open(path) // @@ -48,8 +47,8 @@ import ( // handling if unsafePath has already been scoped within the rootfs (this is // needed for a lot of runc callers and fixing this would require reworking a // lot of path logic). -func MkdirAllInRoot(root, unsafePath string, mode os.FileMode) (*os.File, error) { - unsafePath, err := hallucinateUnsafePath(root, unsafePath) +func MkdirAllInRoot(root *os.File, unsafePath string, mode os.FileMode) (*os.File, error) { + unsafePath, err := hallucinateUnsafePath(root.Name(), unsafePath) if err != nil { return nil, fmt.Errorf("failed to construct hallucinated target path: %w", err) } @@ -67,13 +66,7 @@ func MkdirAllInRoot(root, unsafePath string, mode os.FileMode) (*os.File, error) mode &= 0o1777 } - rootDir, err := os.OpenFile(root, unix.O_DIRECTORY|unix.O_CLOEXEC, 0) - if err != nil { - return nil, fmt.Errorf("open root handle: %w", err) - } - defer rootDir.Close() - return retryEAGAIN(func() (*os.File, error) { - return pathrs.MkdirAllHandle(rootDir, unsafePath, mode) + return pathrs.MkdirAllHandle(root, unsafePath, mode) }) } diff --git a/internal/pathrs/root_pathrslite.go b/internal/pathrs/root_pathrslite.go index 51db77440..0ddabf804 100644 --- a/internal/pathrs/root_pathrslite.go +++ b/internal/pathrs/root_pathrslite.go @@ -28,11 +28,11 @@ import ( ) // OpenInRoot opens the given path inside the root with the provided flags. It -// is effectively shorthand for [securejoin.OpenInRoot] followed by +// is effectively shorthand for [securejoin.OpenatInRoot] followed by // [securejoin.Reopen]. -func OpenInRoot(root, subpath string, flags int) (*os.File, error) { +func OpenInRoot(root *os.File, subpath string, flags int) (*os.File, error) { handle, err := retryEAGAIN(func() (*os.File, error) { - return pathrs.OpenInRoot(root, subpath) + return pathrs.OpenatInRoot(root, subpath) }) if err != nil { return nil, err @@ -47,7 +47,7 @@ func OpenInRoot(root, subpath string, flags int) (*os.File, error) { // open(O_CREAT|O_NOFOLLOW) semantics. If you want the creation to use O_EXCL, // include it in the passed flags. The fileMode argument uses unix.* mode bits, // *not* os.FileMode. -func CreateInRoot(root, subpath string, flags int, fileMode uint32) (*os.File, error) { +func CreateInRoot(root *os.File, subpath string, flags int, fileMode uint32) (*os.File, error) { dirFd, filename, err := MkdirAllParentInRoot(root, subpath, 0o755) if err != nil { return nil, err @@ -63,5 +63,5 @@ func CreateInRoot(root, subpath string, flags int, fileMode uint32) (*os.File, e if err != nil { return nil, err } - return os.NewFile(uintptr(fd), root+"/"+subpath), nil + return os.NewFile(uintptr(fd), root.Name()+"/"+subpath), nil } diff --git a/libcontainer/criu_linux.go b/libcontainer/criu_linux.go index 488095cde..511389af1 100644 --- a/libcontainer/criu_linux.go +++ b/libcontainer/criu_linux.go @@ -553,10 +553,16 @@ func isOnTmpfs(path string, mounts []*configs.Mount) bool { // This function also creates missing mountpoints as long as they // are not on top of a tmpfs, as CRIU will restore tmpfs content anyway. func (c *Container) prepareCriuRestoreMounts(mounts []*configs.Mount) error { + rootFd, err := os.OpenFile(c.config.Rootfs, unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_PATH, 0) + if err != nil { + return fmt.Errorf("open rootfs handle: %w", err) + } + defer rootFd.Close() + umounts := []string{} defer func() { for _, u := range umounts { - mntFile, err := pathrs.OpenInRoot(c.config.Rootfs, u, unix.O_PATH) + mntFile, err := pathrs.OpenInRoot(rootFd, u, unix.O_PATH) if err != nil { logrus.Warnf("Error during cleanup unmounting %s: open handle: %v", u, err) continue @@ -590,7 +596,7 @@ func (c *Container) prepareCriuRestoreMounts(mounts []*configs.Mount) error { continue } me := mountEntry{Mount: m} - if err := me.createOpenMountpoint(c.config.Rootfs); err != nil { + if err := me.createOpenMountpoint(rootFd); err != nil { return fmt.Errorf("create criu restore mountpoint for %s mount: %w", me.Destination, err) } if me.dstFile != nil { diff --git a/libcontainer/rootfs_linux.go b/libcontainer/rootfs_linux.go index 000646084..57bf633a4 100644 --- a/libcontainer/rootfs_linux.go +++ b/libcontainer/rootfs_linux.go @@ -35,7 +35,7 @@ const defaultMountFlags = unix.MS_NOEXEC | unix.MS_NOSUID | unix.MS_NODEV // mountConfig contains mount data not specific to a mount point. type mountConfig struct { - root string + root *os.File label string cgroup2Path string rootlessCgroups bool @@ -160,8 +160,17 @@ func prepareRootfs(pipe *syncSocket, iConfig *initConfig) (err error) { return fmt.Errorf("error preparing rootfs: %w", err) } + // Pre-open rootfs. NOTE that if we need to re-enable support for mounting + // on top of container root (see issue 5070), we will need to reopen rootFd + // after such mounts. + rootFd, err := os.OpenFile(config.Rootfs, unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_PATH, 0) + if err != nil { + return fmt.Errorf("open rootfs handle: %w", err) + } + defer rootFd.Close() + mountConfig := &mountConfig{ - root: config.Rootfs, + root: rootFd, label: config.MountLabel, cgroup2Path: iConfig.Cgroup2Path, rootlessCgroups: config.RootlessCgroups, @@ -175,7 +184,7 @@ func prepareRootfs(pipe *syncSocket, iConfig *initConfig) (err error) { setupDev := needsSetupDev(config) if setupDev { - if err := createDevices(config); err != nil { + if err := createDevices(config, rootFd); err != nil { return fmt.Errorf("error creating device nodes: %w", err) } if err := setupPtmx(config); err != nil { @@ -225,6 +234,7 @@ func prepareRootfs(pipe *syncSocket, iConfig *initConfig) (err error) { if err != nil { return fmt.Errorf("error jailing process inside rootfs: %w", err) } + rootFd.Close() // Apply root mount propagation flags. // This must be done after pivot_root/chroot because the mount propagation flag is applied @@ -370,7 +380,7 @@ func mountCgroupV1(m mountEntry, c *mountConfig) error { // symlink(2) is very dumb, it will just shove the path into // the link and doesn't do any checks or relative path // conversion. Also, don't error out if the cgroup already exists. - if err := os.Symlink(mc, filepath.Join(c.root, m.Destination, ss)); err != nil && !errors.Is(err, os.ErrExist) { + if err := os.Symlink(mc, filepath.Join(c.root.Name(), m.Destination, ss)); err != nil && !errors.Is(err, os.ErrExist) { return err } } @@ -434,15 +444,22 @@ func doTmpfsCopyUp(m mountEntry, mountLabel string) (Err error) { } defer tmpDirFile.Close() + hostRootFd, err := os.OpenFile("/", unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_PATH, 0) + if err != nil { + return fmt.Errorf("tmpcopyup: open host root: %w", err) + } + defer hostRootFd.Close() + // Configure the *host* tmpdir as if it's the container mount. We change // m.dstFile since we are going to mount *on the host*. hostMount := mountEntry{ Mount: m.Mount, dstFile: tmpDirFile, } - if err := hostMount.mountPropagate("/", mountLabel); err != nil { + if err := hostMount.mountPropagate(hostRootFd, mountLabel); err != nil { return err } + hostRootFd.Close() defer func() { if Err != nil { if err := unmount(tmpDir, unix.MNT_DETACH); err != nil { @@ -511,9 +528,10 @@ func statfsToMountFlags(st unix.Statfs_t) int { return flags } -func (m *mountEntry) createOpenMountpoint(rootfs string) (Err error) { +func (m *mountEntry) createOpenMountpoint(root *os.File) (Err error) { + rootfs := root.Name() unsafePath := pathrs.LexicallyStripRoot(rootfs, m.Destination) - dstFile, err := pathrs.OpenInRoot(rootfs, unsafePath, unix.O_PATH) + dstFile, err := pathrs.OpenInRoot(root, unsafePath, unix.O_PATH) defer func() { if dstFile != nil && Err != nil { _ = dstFile.Close() @@ -550,9 +568,9 @@ func (m *mountEntry) createOpenMountpoint(rootfs string) (Err error) { dstIsFile = !fi.IsDir() } if dstIsFile { - dstFile, err = pathrs.CreateInRoot(rootfs, unsafePath, unix.O_CREAT|unix.O_EXCL|unix.O_NOFOLLOW, 0o644) + dstFile, err = pathrs.CreateInRoot(root, unsafePath, unix.O_CREAT|unix.O_EXCL|unix.O_NOFOLLOW, 0o644) } else { - dstFile, err = pathrs.MkdirAllInRoot(rootfs, unsafePath, 0o755) + dstFile, err = pathrs.MkdirAllInRoot(root, unsafePath, 0o755) } if err != nil { return fmt.Errorf("make mountpoint %q: %w", m.Destination, err) @@ -598,7 +616,7 @@ func mountToRootfs(c *mountConfig, m mountEntry) error { // has been a "fun" attack scenario in the past. // TODO: This won't be necessary once we switch to libpathrs and we can // stop all of these symlink-exchange attacks. - rootfs := c.root + rootfs := c.root.Name() dest := filepath.Clean(m.Destination) if !pathrs.IsLexicallyInRoot(rootfs, dest) { // Do not use securejoin as it resolves symlinks. @@ -931,7 +949,7 @@ func reOpenDevNull() error { } // Create the device nodes in the container. -func createDevices(config *configs.Config) error { +func createDevices(config *configs.Config, rootFd *os.File) error { useBindMount := userns.RunningInUserNS() || config.Namespaces.Contains(configs.NEWUSER) for _, node := range config.Devices { @@ -942,7 +960,7 @@ func createDevices(config *configs.Config) error { // containers running in a user namespace are not allowed to mknod // devices so we can just bind mount it from the host. - if err := createDeviceNode(config.Rootfs, node, useBindMount); err != nil { + if err := createDeviceNode(rootFd, node, useBindMount); err != nil { return err } } @@ -962,12 +980,12 @@ func bindMountDeviceNode(destDir *os.File, destName string, node *devices.Device } // Creates the device node in the rootfs of the container. -func createDeviceNode(rootfs string, node *devices.Device, bind bool) error { +func createDeviceNode(rootFd *os.File, node *devices.Device, bind bool) error { if node.Path == "" { // The node only exists for cgroup reasons, ignore it here. return nil } - destDir, destName, err := pathrs.MkdirAllParentInRoot(rootfs, node.Path, 0o755) + destDir, destName, err := pathrs.MkdirAllParentInRoot(rootFd, node.Path, 0o755) if err != nil { return fmt.Errorf("mkdir parent of device inode %q: %w", node.Path, err) } @@ -1360,16 +1378,17 @@ func maskPaths(paths []string, mountLabel string) error { return nil } -func reopenAfterMount(rootfs string, f *os.File, flags int) (_ *os.File, Err error) { +func reopenAfterMount(rootFd, f *os.File, flags int) (_ *os.File, Err error) { fullPath, err := procfs.ProcSelfFdReadlink(f) if err != nil { return nil, fmt.Errorf("get full path: %w", err) } + rootfs := rootFd.Name() if !pathrs.IsLexicallyInRoot(rootfs, fullPath) { return nil, fmt.Errorf("mountpoint %q is outside of rootfs %q", fullPath, rootfs) } unsafePath := pathrs.LexicallyStripRoot(rootfs, fullPath) - reopened, err := pathrs.OpenInRoot(rootfs, unsafePath, flags) + reopened, err := pathrs.OpenInRoot(rootFd, unsafePath, flags) if err != nil { return nil, fmt.Errorf("re-open mountpoint %q: %w", unsafePath, err) } @@ -1399,7 +1418,7 @@ func reopenAfterMount(rootfs string, f *os.File, flags int) (_ *os.File, Err err // Do the mount operation followed by additional mounts required to take care // of propagation flags. This will always be scoped inside the container rootfs. -func (m *mountEntry) mountPropagate(rootfs, mountLabel string) error { +func (m *mountEntry) mountPropagate(rootFd *os.File, mountLabel string) error { var ( data = label.FormatMountLabel(m.Data, mountLabel) flags = m.Flags @@ -1425,7 +1444,7 @@ func (m *mountEntry) mountPropagate(rootfs, mountLabel string) error { // // TODO: Use move_mount(2) on newer kernels so that this is no longer // necessary on modern systems. - newDstFile, err := reopenAfterMount(rootfs, m.dstFile, unix.O_PATH) + newDstFile, err := reopenAfterMount(rootFd, m.dstFile, unix.O_PATH) if err != nil { return fmt.Errorf("reopen mountpoint after mount: %w", err) }