diff --git a/cgroups/fs/apply_raw.go b/cgroups/fs/apply_raw.go index 64b9bf452..b272182b9 100644 --- a/cgroups/fs/apply_raw.go +++ b/cgroups/fs/apply_raw.go @@ -30,6 +30,7 @@ var ( "freezer": &FreezerGroup{}, } CgroupProcesses = "cgroup.procs" + HugePageSizes, _ = cgroups.GetHugePageSize() ) type subsystem interface { diff --git a/cgroups/fs/hugetlb.go b/cgroups/fs/hugetlb.go index 277e87fe8..4b8264933 100644 --- a/cgroups/fs/hugetlb.go +++ b/cgroups/fs/hugetlb.go @@ -3,6 +3,10 @@ package fs import ( + "fmt" + "strconv" + "strings" + "github.com/docker/libcontainer/cgroups" "github.com/docker/libcontainer/configs" ) @@ -11,14 +15,25 @@ type HugetlbGroup struct { } func (s *HugetlbGroup) Apply(d *data) error { - // we just want to join this group even though we don't set anything - if _, err := d.join("hugetlb"); err != nil && !cgroups.IsNotFound(err) { + dir, err := d.join("hugetlb") + if err != nil && !cgroups.IsNotFound(err) { return err } + + if err := s.Set(dir, d.c); err != nil { + return err + } + return nil } func (s *HugetlbGroup) Set(path string, cgroup *configs.Cgroup) error { + for _, hugetlb := range cgroup.HugetlbLimit { + if err := writeFile(path, strings.Join([]string{"hugetlb", hugetlb.Pagesize, "limit_in_bytes"}, "."), strconv.Itoa(hugetlb.Limit)); err != nil { + return err + } + } + return nil } @@ -27,5 +42,31 @@ func (s *HugetlbGroup) Remove(d *data) error { } func (s *HugetlbGroup) GetStats(path string, stats *cgroups.Stats) error { + hugetlbStats := cgroups.HugetlbStats{} + for _, pageSize := range HugePageSizes { + usage := strings.Join([]string{"hugetlb", pageSize, "usage_in_bytes"}, ".") + value, err := getCgroupParamUint(path, usage) + if err != nil { + return fmt.Errorf("failed to parse %s - %v", usage, err) + } + hugetlbStats.Usage = value + + maxUsage := strings.Join([]string{"hugetlb", pageSize, "max_usage_in_bytes"}, ".") + value, err = getCgroupParamUint(path, maxUsage) + if err != nil { + return fmt.Errorf("failed to parse %s - %v", maxUsage, err) + } + hugetlbStats.MaxUsage = value + + failcnt := strings.Join([]string{"hugetlb", pageSize, "failcnt"}, ".") + value, err = getCgroupParamUint(path, failcnt) + if err != nil { + return fmt.Errorf("failed to parse %s - %v", failcnt, err) + } + hugetlbStats.Failcnt = value + + stats.HugetlbStats[pageSize] = hugetlbStats + } + return nil } diff --git a/cgroups/fs/hugetlb_test.go b/cgroups/fs/hugetlb_test.go new file mode 100644 index 000000000..7dc0e7539 --- /dev/null +++ b/cgroups/fs/hugetlb_test.go @@ -0,0 +1,138 @@ +package fs + +import ( + "strconv" + "strings" + "testing" + + "github.com/docker/libcontainer/cgroups" + "github.com/docker/libcontainer/configs" +) + +const ( + hugetlbUsageContents = "128\n" + hugetlbMaxUsageContents = "256\n" + hugetlbFailcnt = "100\n" +) + +var ( + hugePageSize, _ = cgroups.GetHugePageSize() + usage = strings.Join([]string{"hugetlb", hugePageSize[0], "usage_in_bytes"}, ".") + limit = strings.Join([]string{"hugetlb", hugePageSize[0], "limit_in_bytes"}, ".") + maxUsage = strings.Join([]string{"hugetlb", hugePageSize[0], "max_usage_in_bytes"}, ".") + failcnt = strings.Join([]string{"hugetlb", hugePageSize[0], "failcnt"}, ".") +) + +func TestHugetlbSetHugetlb(t *testing.T) { + helper := NewCgroupTestUtil("hugetlb", t) + defer helper.cleanup() + + const ( + hugetlbBefore = 256 + hugetlbAfter = 512 + ) + + helper.writeFileContents(map[string]string{ + limit: strconv.Itoa(hugetlbBefore), + }) + + helper.CgroupData.c.HugetlbLimit = []*configs.HugepageLimit{ + { + Pagesize: hugePageSize[0], + Limit: hugetlbAfter, + }, + } + hugetlb := &HugetlbGroup{} + if err := hugetlb.Set(helper.CgroupPath, helper.CgroupData.c); err != nil { + t.Fatal(err) + } + + value, err := getCgroupParamUint(helper.CgroupPath, limit) + if err != nil { + t.Fatalf("Failed to parse %s - %s", limit, err) + } + if value != hugetlbAfter { + t.Fatalf("Set hugetlb.limit_in_bytes failed. Expected: %v, Got: %v", hugetlbAfter, value) + } +} + +func TestHugetlbStats(t *testing.T) { + helper := NewCgroupTestUtil("hugetlb", t) + defer helper.cleanup() + helper.writeFileContents(map[string]string{ + usage: hugetlbUsageContents, + maxUsage: hugetlbMaxUsageContents, + failcnt: hugetlbFailcnt, + }) + + hugetlb := &HugetlbGroup{} + actualStats := *cgroups.NewStats() + err := hugetlb.GetStats(helper.CgroupPath, &actualStats) + if err != nil { + t.Fatal(err) + } + expectedStats := cgroups.HugetlbStats{Usage: 128, MaxUsage: 256, Failcnt: 100} + expectHugetlbStatEquals(t, expectedStats, actualStats.HugetlbStats[hugePageSize[0]]) +} + +func TestHugetlbStatsNoUsageFile(t *testing.T) { + helper := NewCgroupTestUtil("hugetlb", t) + defer helper.cleanup() + helper.writeFileContents(map[string]string{ + maxUsage: hugetlbMaxUsageContents, + }) + + hugetlb := &HugetlbGroup{} + actualStats := *cgroups.NewStats() + err := hugetlb.GetStats(helper.CgroupPath, &actualStats) + if err == nil { + t.Fatal("Expected failure") + } +} + +func TestHugetlbStatsNoMaxUsageFile(t *testing.T) { + helper := NewCgroupTestUtil("hugetlb", t) + defer helper.cleanup() + helper.writeFileContents(map[string]string{ + usage: hugetlbUsageContents, + }) + + hugetlb := &HugetlbGroup{} + actualStats := *cgroups.NewStats() + err := hugetlb.GetStats(helper.CgroupPath, &actualStats) + if err == nil { + t.Fatal("Expected failure") + } +} + +func TestHugetlbStatsBadUsageFile(t *testing.T) { + helper := NewCgroupTestUtil("hugetlb", t) + defer helper.cleanup() + helper.writeFileContents(map[string]string{ + usage: "bad", + maxUsage: hugetlbMaxUsageContents, + }) + + hugetlb := &HugetlbGroup{} + actualStats := *cgroups.NewStats() + err := hugetlb.GetStats(helper.CgroupPath, &actualStats) + if err == nil { + t.Fatal("Expected failure") + } +} + +func TestHugetlbStatsBadMaxUsageFile(t *testing.T) { + helper := NewCgroupTestUtil("hugetlb", t) + defer helper.cleanup() + helper.writeFileContents(map[string]string{ + usage: hugetlbUsageContents, + maxUsage: "bad", + }) + + hugetlb := &HugetlbGroup{} + actualStats := *cgroups.NewStats() + err := hugetlb.GetStats(helper.CgroupPath, &actualStats) + if err == nil { + t.Fatal("Expected failure") + } +} diff --git a/cgroups/fs/stats_util_test.go b/cgroups/fs/stats_util_test.go index 9d8cfdb10..19226207a 100644 --- a/cgroups/fs/stats_util_test.go +++ b/cgroups/fs/stats_util_test.go @@ -72,6 +72,13 @@ func expectThrottlingDataEquals(t *testing.T, expected, actual cgroups.Throttlin } } +func expectHugetlbStatEquals(t *testing.T, expected, actual cgroups.HugetlbStats) { + if expected != actual { + logrus.Printf("Expected hugetlb stats %v but found %v\n", expected, actual) + t.Fail() + } +} + func expectMemoryStatEquals(t *testing.T, expected, actual cgroups.MemoryStats) { expectMemoryDataEquals(t, expected.Usage, actual.Usage) expectMemoryDataEquals(t, expected.SwapUsage, actual.SwapUsage) diff --git a/cgroups/stats.go b/cgroups/stats.go index ccfe652cc..bda32b20c 100644 --- a/cgroups/stats.go +++ b/cgroups/stats.go @@ -68,13 +68,25 @@ type BlkioStats struct { SectorsRecursive []BlkioStatEntry `json:"sectors_recursive,omitempty"` } +type HugetlbStats struct { + // current res_counter usage for hugetlb + Usage uint64 `json:"usage,omitempty"` + // maximum usage ever recorded. + MaxUsage uint64 `json:"max_usage,omitempty"` + // number of times htgetlb usage allocation failure. + Failcnt uint64 `json:"failcnt"` +} + type Stats struct { CpuStats CpuStats `json:"cpu_stats,omitempty"` MemoryStats MemoryStats `json:"memory_stats,omitempty"` BlkioStats BlkioStats `json:"blkio_stats,omitempty"` + // the map is in the format "size of hugepage: stats of the hugepage" + HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"` } func NewStats() *Stats { memoryStats := MemoryStats{Stats: make(map[string]uint64)} - return &Stats{MemoryStats: memoryStats} + hugetlbStats := make(map[string]HugetlbStats) + return &Stats{MemoryStats: memoryStats, HugetlbStats: hugetlbStats} } diff --git a/cgroups/systemd/apply_systemd.go b/cgroups/systemd/apply_systemd.go index 2a0427211..98e9a5d38 100644 --- a/cgroups/systemd/apply_systemd.go +++ b/cgroups/systemd/apply_systemd.go @@ -223,6 +223,9 @@ func (m *Manager) Apply(pid int) error { return err } + if err := joinHugetlb(c, pid); err != nil { + return err + } // FIXME: Systemd does have `BlockIODeviceWeight` property, but we got problem // using that (at least on systemd 208, see https://github.com/docker/libcontainer/pull/354), // so use fs work around for now. @@ -538,3 +541,13 @@ func joinBlkio(c *configs.Cgroup, pid int) error { return nil } + +func joinHugetlb(c *configs.Cgroup, pid int) error { + path, err := join(c, "hugetlb", pid) + if err != nil && !cgroups.IsNotFound(err) { + return err + } + + hugetlb := subsystems["hugetlb"] + return hugetlb.Set(path, c) +} diff --git a/cgroups/utils.go b/cgroups/utils.go index 548688399..a64818307 100644 --- a/cgroups/utils.go +++ b/cgroups/utils.go @@ -14,6 +14,7 @@ import ( "time" "github.com/docker/docker/pkg/mount" + "github.com/docker/docker/pkg/units" ) // https://www.kernel.org/doc/Documentation/cgroups/cgroups.txt @@ -238,3 +239,23 @@ func RemovePaths(paths map[string]string) (err error) { } return fmt.Errorf("Failed to remove paths: %s", paths) } + +func GetHugePageSize() ([]string, error) { + var pageSizes []string + sizeList := []string{"B", "kB", "MB", "GB", "TB", "PB"} + files, err := ioutil.ReadDir("/sys/kernel/mm/hugepages") + if err != nil { + return pageSizes, err + } + for _, st := range files { + nameArray := strings.Split(st.Name(), "-") + pageSize, err := units.RAMInBytes(nameArray[1]) + if err != nil { + return []string{}, err + } + sizeString := units.CustomSize("%g%s", float64(pageSize), 1024.0, sizeList) + pageSizes = append(pageSizes, sizeString) + } + + return pageSizes, nil +} diff --git a/configs/cgroup.go b/configs/cgroup.go index 55a81ded2..88b2a71d8 100644 --- a/configs/cgroup.go +++ b/configs/cgroup.go @@ -78,6 +78,9 @@ type Cgroup struct { // set the freeze value for the process Freezer FreezerState `json:"freezer"` + // Hugetlb limit (in bytes) + HugetlbLimit []*HugepageLimit `json:"hugetlb_limit"` + // Parent slice to use for systemd TODO: remove in favor or parent Slice string `json:"slice"` diff --git a/configs/hugepage_limit.go b/configs/hugepage_limit.go new file mode 100644 index 000000000..1cce8d09b --- /dev/null +++ b/configs/hugepage_limit.go @@ -0,0 +1,9 @@ +package configs + +type HugepageLimit struct { + // which type of hugepage to limit. + Pagesize string `json:"page_size"` + + // usage limit for hugepage. + Limit int `json:"limit"` +}