From 705b6cc723f13c0b7265e6457ea7672736a5cce9 Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Mon, 8 Feb 2021 13:05:54 +0000 Subject: [PATCH 1/2] Re-create mountpoints during restore runc was already re-creating mountpoints before calling CRIU to restore a container. But mountpoints inside a bind mount were not re-created. During initial container creation runc will mount bind mounts and then create the necessary mountpoints for further mounts inside those bind mounts. If, for example, one of the bind mounts is a tmpfs and empty before restore, CRIU will fail re-mounting all mounts because the mountpoints in the bind mounted tmpfs no longer exist. CRIU expects all mount points to exist as during checkpointing. This changes runc to mount bind mounts after mountpoint creation to ensure nested bind mounts have their mountpoints created before CRIU does the restore. Signed-off-by: Adrian Reber --- libcontainer/container_linux.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/libcontainer/container_linux.go b/libcontainer/container_linux.go index 3dca29e4c..38e5a71a4 100644 --- a/libcontainer/container_linux.go +++ b/libcontainer/container_linux.go @@ -1235,11 +1235,38 @@ func (c *linuxContainer) prepareCriuRestoreMounts(mounts []*configs.Mount) error // Now go through all mounts and create the mountpoints // if the mountpoints are not on a tmpfs, as CRIU will // restore the complete tmpfs content from its checkpoint. + umounts := []string{} + defer func() { + for _, u := range umounts { + if e := unix.Unmount(u, unix.MNT_DETACH); e != nil { + if e != unix.EINVAL { + // Ignore EINVAL as it means 'target is not a mount point.' + // It probably has already been unmounted. + logrus.Warnf("Error during cleanup unmounting of %q (%v)", u, e) + } + } + } + }() for _, m := range mounts { if !isPathInPrefixList(m.Destination, tmpfs) { if err := c.makeCriuRestoreMountpoints(m); err != nil { return err } + // If the mount point is a bind mount, we need to mount + // it now so that runc can create the necessary mount + // points for mounts in bind mounts. + // This also happens during initial container creation. + // Without this CRIU restore will fail + // See: https://github.com/opencontainers/runc/issues/2748 + // It is also not necessary to order the mount points + // because during initial container creation mounts are + // set up in the order they are configured. + if m.Device == "bind" { + if err := unix.Mount(m.Source, m.Destination, "", unix.MS_BIND|unix.MS_REC, ""); err != nil { + return errorsf.Wrapf(err, "unable to bind mount %q to %q", m.Source, m.Destination) + } + umounts = append(umounts, m.Destination) + } } } return nil From 051646a3b09ca48aac4f9e51fa0d5f80b7f086cd Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Mon, 8 Feb 2021 13:20:02 +0000 Subject: [PATCH 2/2] tests: test nested bind mount restore This adds a checkpoint/restore test to ensure that nested bind mount mountpoints are correctly re-created. Signed-off-by: Adrian Reber --- tests/integration/checkpoint.bats | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/integration/checkpoint.bats b/tests/integration/checkpoint.bats index e45538380..4e7153a5d 100644 --- a/tests/integration/checkpoint.bats +++ b/tests/integration/checkpoint.bats @@ -315,3 +315,45 @@ function simple_cr() { unlink "$tmp" test -f ./work-dir/"$tmplog2" && unlink ./work-dir/"$tmplog2" } + +@test "checkpoint and restore with nested bind mounts" { + bind1=$(mktemp -d -p .) + bind2=$(mktemp -d -p .) + update_config ' .mounts += [{ + type: "bind", + source: "'"$bind1"'", + destination: "/test", + options: ["rw", "bind"] + }, + { + type: "bind", + source: "'"$bind2"'", + destination: "/test/for/nested", + options: ["rw", "bind"] + }]' + + runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox + [ "$status" -eq 0 ] + + testcontainer test_busybox running + + # checkpoint the running container + runc --criu "$CRIU" checkpoint --work-path ./work-dir test_busybox + grep -B 5 Error ./work-dir/dump.log || true + [ "$status" -eq 0 ] + + # after checkpoint busybox is no longer running + testcontainer test_busybox checkpointed + + # cleanup mountpoints created by runc during creation + # the mountpoints should be recreated during restore - that is the actual thing tested here + rm -rf "${bind1:?}"/* + + # restore from checkpoint + runc --criu "$CRIU" restore -d --work-path ./work-dir --console-socket "$CONSOLE_SOCKET" test_busybox + grep -B 5 Error ./work-dir/restore.log || true + [ "$status" -eq 0 ] + + # busybox should be back up and running + testcontainer test_busybox running +}