diff --git a/libcontainer/cgroups/cgroups.go b/libcontainer/cgroups/cgroups.go index a16a68e97..c418fbae9 100644 --- a/libcontainer/cgroups/cgroups.go +++ b/libcontainer/cgroups/cgroups.go @@ -48,4 +48,7 @@ type Manager interface { // Whether the cgroup path exists or not Exists() bool + + // OOMKillCount reports OOM kill count for the cgroup. + OOMKillCount() (uint64, error) } diff --git a/libcontainer/cgroups/fs/fs.go b/libcontainer/cgroups/fs/fs.go index a42ce4535..bede91e2f 100644 --- a/libcontainer/cgroups/fs/fs.go +++ b/libcontainer/cgroups/fs/fs.go @@ -9,6 +9,7 @@ import ( "sync" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils" "github.com/pkg/errors" @@ -421,3 +422,11 @@ func (m *manager) GetFreezerState() (configs.FreezerState, error) { func (m *manager) Exists() bool { return cgroups.PathExists(m.Path("devices")) } + +func OOMKillCount(path string) (uint64, error) { + return fscommon.GetValueByKey(path, "memory.oom_control", "oom_kill") +} + +func (m *manager) OOMKillCount() (uint64, error) { + return OOMKillCount(m.Path("memory")) +} diff --git a/libcontainer/cgroups/fs2/fs2.go b/libcontainer/cgroups/fs2/fs2.go index 3f0b9e0d7..733f9dcfd 100644 --- a/libcontainer/cgroups/fs2/fs2.go +++ b/libcontainer/cgroups/fs2/fs2.go @@ -257,3 +257,11 @@ func (m *manager) GetFreezerState() (configs.FreezerState, error) { func (m *manager) Exists() bool { return cgroups.PathExists(m.dirPath) } + +func OOMKillCount(path string) (uint64, error) { + return fscommon.GetValueByKey(path, "memory.events", "oom_kill") +} + +func (m *manager) OOMKillCount() (uint64, error) { + return OOMKillCount(m.dirPath) +} diff --git a/libcontainer/cgroups/systemd/v1.go b/libcontainer/cgroups/systemd/v1.go index 2ad1347e5..59a8b1e72 100644 --- a/libcontainer/cgroups/systemd/v1.go +++ b/libcontainer/cgroups/systemd/v1.go @@ -450,3 +450,7 @@ func (m *legacyManager) GetFreezerState() (configs.FreezerState, error) { func (m *legacyManager) Exists() bool { return cgroups.PathExists(m.Path("devices")) } + +func (m *legacyManager) OOMKillCount() (uint64, error) { + return fs.OOMKillCount(m.Path("memory")) +} diff --git a/libcontainer/cgroups/systemd/v2.go b/libcontainer/cgroups/systemd/v2.go index 6f1422659..65abc0592 100644 --- a/libcontainer/cgroups/systemd/v2.go +++ b/libcontainer/cgroups/systemd/v2.go @@ -495,3 +495,7 @@ func (m *unifiedManager) GetFreezerState() (configs.FreezerState, error) { func (m *unifiedManager) Exists() bool { return cgroups.PathExists(m.path) } + +func (m *unifiedManager) OOMKillCount() (uint64, error) { + return fs2.OOMKillCount(m.path) +} diff --git a/libcontainer/container_linux_test.go b/libcontainer/container_linux_test.go index 73915997c..2a9af6ad2 100644 --- a/libcontainer/container_linux_test.go +++ b/libcontainer/container_linux_test.go @@ -55,6 +55,10 @@ func (m *mockCgroupManager) Exists() bool { return err == nil } +func (m *mockCgroupManager) OOMKillCount() (uint64, error) { + return 0, nil +} + func (m *mockCgroupManager) GetPaths() map[string]string { return m.paths } diff --git a/libcontainer/process_linux.go b/libcontainer/process_linux.go index 834268b99..7e403c6af 100644 --- a/libcontainer/process_linux.go +++ b/libcontainer/process_linux.go @@ -321,6 +321,24 @@ func (p *initProcess) start() (retErr error) { } defer func() { if retErr != nil { + // init might be killed by the kernel's OOM killer. + oom, err := p.manager.OOMKillCount() + if err != nil { + logrus.WithError(err).Warn("unable to get oom kill count") + } else if oom > 0 { + // Does not matter what the particular error was, + // its cause is most probably OOM, so report that. + const oomError = "container init was OOM-killed (memory limit too low?)" + + if logrus.GetLevel() >= logrus.DebugLevel { + // Only show the original error if debug is set, + // as it is not generally very useful. + retErr = newSystemErrorWithCause(retErr, oomError) + } else { + retErr = newSystemError(errors.New(oomError)) + } + } + // terminate the process to ensure we can remove cgroups if err := ignoreTerminateErrors(p.terminate()); err != nil { logrus.WithError(err).Warn("unable to terminate initProcess") diff --git a/tests/integration/cgroups.bats b/tests/integration/cgroups.bats index a319ed042..38d063047 100644 --- a/tests/integration/cgroups.bats +++ b/tests/integration/cgroups.bats @@ -79,7 +79,7 @@ function setup() { runc run -d --console-socket "$CONSOLE_SOCKET" test_cgroups_permissions [ "$status" -eq 1 ] - [[ ${lines[0]} == *"permission denied"* ]] + [[ "$output" == *"applying cgroup configuration"*"permission denied"* ]] } @test "runc create (rootless + limits + no cgrouppath + no permission) fails with informative error" { @@ -92,7 +92,8 @@ function setup() { runc run -d --console-socket "$CONSOLE_SOCKET" test_cgroups_permissions [ "$status" -eq 1 ] - [[ ${lines[0]} == *"rootless needs no limits + no cgrouppath when no permission is granted for cgroups"* ]] || [[ ${lines[0]} == *"cannot set pids limit: container could not join or create cgroup"* ]] + [[ "$output" == *"rootless needs no limits + no cgrouppath when no permission is granted for cgroups"* ]] || + [[ "$output" == *"cannot set pids limit: container could not join or create cgroup"* ]] } @test "runc create (limits + cgrouppath + permission on the cgroup dir) succeeds" {