From fa43a72aba4a1ec54fe830b7919abdb736025c66 Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Mon, 2 Jul 2018 20:22:38 +0000 Subject: [PATCH 1/2] criu: restore into existing namespace when specified Using CRIU to checkpoint and restore a container into an existing network namespace is not possible. If the network namespace is defined like { "type": "network", "path": "/run/netns/test" } there is the expectation that the restored container is again running in the network namespace specified with 'path'. This adds the new CRIU 'external namespace' feature to runc, where during checkpointing that specific namespace is referenced and during restore CRIU tries to restore the container in exactly that namespace. This breaks/fixes current runc behavior. If, without this patch, runc restores a container with such a network namespace definition, it is ignored and CRIU recreates a network namespace without a name. With this patch runc uses the network namespace path (if available) to checkpoint and restore the container in just that network namespace. Restore will now fail if a container was checkpointed with a network namespace path set and if that network namespace path does not exist during restore. runc still falls back to the old behavior if CRIU older than 3.11 is installed. Fixes #1786 Related to https://github.com/projectatomic/libpod/pull/469 Thanks to Andrei Vagin for all the help in getting the interface between CRIU and runc right! Signed-off-by: Adrian Reber --- libcontainer/container_linux.go | 74 ++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/libcontainer/container_linux.go b/libcontainer/container_linux.go index 741334741..9f551fd09 100644 --- a/libcontainer/container_linux.go +++ b/libcontainer/container_linux.go @@ -657,7 +657,7 @@ func (c *linuxContainer) checkCriuFeatures(criuOpts *CriuOpts, rpcOpts *criurpc. Features: criuFeat, } - err := c.criuSwrk(nil, req, criuOpts, false) + err := c.criuSwrk(nil, req, criuOpts, false, nil) if err != nil { logrus.Debugf("%s", err) return fmt.Errorf("CRIU feature check failed") @@ -770,7 +770,7 @@ func (c *linuxContainer) checkCriuVersion(minVersion int) error { Type: &t, } - err := c.criuSwrk(nil, req, nil, false) + err := c.criuSwrk(nil, req, nil, false, nil) if err != nil { return fmt.Errorf("CRIU version check failed: %s", err) } @@ -928,6 +928,33 @@ func (c *linuxContainer) Checkpoint(criuOpts *CriuOpts) error { LazyPages: proto.Bool(criuOpts.LazyPages), } + // If the container is running in a network namespace and has + // a path to the network namespace configured, we will dump + // that network namespace as an external namespace and we + // will expect that the namespace exists during restore. + // This basically means that CRIU will ignore the namespace + // and expect to be setup correctly. + nsPath := c.config.Namespaces.PathOf(configs.NEWNET) + if nsPath != "" { + // For this to work we need at least criu 3.11.0 => 31100. + // As there was already a successful version check we will + // not error out if it fails. runc will just behave as it used + // to do and ignore external network namespaces. + err := c.checkCriuVersion(31100) + if err == nil { + // CRIU expects the information about an external namespace + // like this: --external net[]: + // This is always 'extRootNetNS'. + var netns syscall.Stat_t + err = syscall.Stat(nsPath, &netns) + if err != nil { + return err + } + criuExternal := fmt.Sprintf("net[%d]:extRootNetNS", netns.Ino) + rpcOpts.External = append(rpcOpts.External, criuExternal) + } + } + fcg := c.cgroupManager.GetPaths()["freezer"] if fcg != "" { rpcOpts.FreezeCgroup = proto.String(fcg) @@ -1032,7 +1059,7 @@ func (c *linuxContainer) Checkpoint(criuOpts *CriuOpts) error { } } - err = c.criuSwrk(nil, req, criuOpts, false) + err = c.criuSwrk(nil, req, criuOpts, false, nil) if err != nil { return err } @@ -1076,6 +1103,8 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { c.m.Lock() defer c.m.Unlock() + var extraFiles []*os.File + // TODO(avagin): Figure out how to make this work nicely. CRIU doesn't have // support for unprivileged restore at the moment. if c.config.Rootless { @@ -1150,6 +1179,38 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { }, } + // Same as during checkpointing. If the container has a specific network namespace + // assigned to it, this now expects that the checkpoint will be restored in a + // already created network namespace. + nsPath := c.config.Namespaces.PathOf(configs.NEWNET) + if nsPath != "" { + // For this to work we need at least criu 3.11.0 => 31100. + // As there was already a successful version check we will + // not error out if it fails. runc will just behave as it used + // to do and ignore external network namespaces. + err := c.checkCriuVersion(31100) + if err == nil { + // CRIU wants the information about an existing network namespace + // like this: --inherit-fd fd[]: + // The needs to be the same as during checkpointing. + // We are always using 'extRootNetNS' as the key in this. + netns, err := os.Open(nsPath) + defer netns.Close() + if err != nil { + logrus.Error("If a specific network namespace is defined it must exist: %s", err) + return fmt.Errorf("Requested network namespace %v does not exist", nsPath) + } + inheritFd := new(criurpc.InheritFd) + inheritFd.Key = proto.String("extRootNetNS") + // The offset of four is necessary because 0, 1, 2 and 3 is already + // used by stdin, stdout, stderr, 'criu swrk' socket. + inheritFd.Fd = proto.Int32(int32(4 + len(extraFiles))) + req.Opts.InheritFd = append(req.Opts.InheritFd, inheritFd) + // All open FDs need to be transferred to CRIU via extraFiles + extraFiles = append(extraFiles, netns) + } + } + for _, m := range c.config.Mounts { switch m.Device { case "bind": @@ -1208,7 +1269,7 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { req.Opts.InheritFd = append(req.Opts.InheritFd, inheritFd) } } - return c.criuSwrk(process, req, criuOpts, true) + return c.criuSwrk(process, req, criuOpts, true, extraFiles) } func (c *linuxContainer) criuApplyCgroups(pid int, req *criurpc.CriuReq) error { @@ -1238,7 +1299,7 @@ func (c *linuxContainer) criuApplyCgroups(pid int, req *criurpc.CriuReq) error { return nil } -func (c *linuxContainer) criuSwrk(process *Process, req *criurpc.CriuReq, opts *CriuOpts, applyCgroups bool) error { +func (c *linuxContainer) criuSwrk(process *Process, req *criurpc.CriuReq, opts *CriuOpts, applyCgroups bool, extraFiles []*os.File) error { fds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) if err != nil { return err @@ -1279,6 +1340,9 @@ func (c *linuxContainer) criuSwrk(process *Process, req *criurpc.CriuReq, opts * cmd.Stderr = process.Stderr } cmd.ExtraFiles = append(cmd.ExtraFiles, criuServer) + if extraFiles != nil { + cmd.ExtraFiles = append(cmd.ExtraFiles, extraFiles...) + } if err := cmd.Start(); err != nil { return err From 832ac8a5382a4a3c3db287f584bd5e9e9fe9852f Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Fri, 6 Jul 2018 18:40:18 +0000 Subject: [PATCH 2/2] tests: add external network namespace tests This adds a new CRIU based checkpoint/restore test to check if the restored container runs in the same network namespace as before. Signed-off-by: Adrian Reber --- tests/integration/checkpoint.bats | 65 +++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/integration/checkpoint.bats b/tests/integration/checkpoint.bats index 3aaf03844..e390067fb 100644 --- a/tests/integration/checkpoint.bats +++ b/tests/integration/checkpoint.bats @@ -230,3 +230,68 @@ function teardown() { [ "$status" -eq 0 ] [[ "${output}" == *"ponG Ping"* ]] } + +@test "checkpoint and restore in external network namespace" { + # XXX: currently criu require root containers. + requires criu root + + # check if external_net_ns is supported; only with criu 3.10++ + run ${CRIU} check --feature external_net_ns + if [ "$status" -eq 1 ]; then + # this criu does not support external_net_ns; skip the test + skip "this criu does not support external network namespaces" + fi + + # create a temporary name for the test network namespace + tmp=`mktemp` + rm -f $tmp + ns_name=`basename $tmp` + # create network namespace + ip netns add $ns_name + ns_path=`ip netns add $ns_name 2>&1 | sed -e 's/.*"\(.*\)".*/\1/'` + + ns_inode=`ls -iL $ns_path | awk '{ print $1 }'` + + # not necessary with criu 3.10 any more + sed -i 's;"readonly": true;"readonly": false;' config.json + # tell runc which network namespace to use + sed -i "s;\"type\": \"network\";\"type\": \"network\",\"path\": \"$ns_path\";" config.json + + runc run -d --console-socket $CONSOLE_SOCKET test_busybox + [ "$status" -eq 0 ] + + testcontainer test_busybox running + + for i in `seq 2`; do + # checkpoint the running container; this automatically tells CRIU to + # handle the network namespace defined in config.json as an external + runc --criu "$CRIU" checkpoint --work-path ./work-dir test_busybox + ret=$? + # if you are having problems getting criu to work uncomment the following dump: + #cat /run/opencontainer/containers/test_busybox/criu.work/dump.log + cat ./work-dir/dump.log | grep -B 5 Error || true + [ "$ret" -eq 0 ] + + # after checkpoint busybox is no longer running + runc state test_busybox + [ "$status" -ne 0 ] + + # restore from checkpoint; this should restore the container into the existing network namespace + runc --criu "$CRIU" restore -d --work-path ./work-dir --console-socket $CONSOLE_SOCKET test_busybox + ret=$? + cat ./work-dir/restore.log | grep -B 5 Error || true + [ "$ret" -eq 0 ] + + # busybox should be back up and running + testcontainer test_busybox running + + # container should be running in same network namespace as before + pid=`__runc state test_busybox | jq '.pid'` + ns_inode_new=`readlink /proc/$pid/ns/net | sed -e 's/.*\[\(.*\)\]/\1/'` + echo "old network namespace inode $ns_inode" + echo "new network namespace inode $ns_inode_new" + [ "$ns_inode" -eq "$ns_inode_new" ] + done + ip netns del $ns_name +} +