From 39d4c8d5f969eb0623d72c744c2a83ef89d1ea2d Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Tue, 5 Oct 2021 17:18:03 -0700 Subject: [PATCH] libct/cg: lazy init for HugePageSizes I have noticed that libct/cg/fs allocates 8K during init on every runc execution: > init github.com/opencontainers/runc/libcontainer/cgroups/fs @1.5 ms, 0.028 ms clock, 8512 bytes, 13 allocs Apparently this is caused by global HugePageSizes variable init, which is only used from GetStats (i.e. it is never used by runc itself). Remove it, and use HugePageSizes() directly instead. Make it init-once, so that GetStats (which, I guess, is periodically called by kubernetes) does not re-read huge page sizes over and over. This also removes 12 allocs and 8K from libct/cg/fs init section: > $ time GODEBUG=inittrace=1 ./runc --help 2>&1 | grep cgroups/fs > init github.com/opencontainers/runc/libcontainer/cgroups/fs @1.5 ms, 0.003 ms clock, 16 bytes, 1 allocs Signed-off-by: Kir Kolyshkin --- libcontainer/cgroups/fs/fs.go | 35 +++++++++++-------------- libcontainer/cgroups/fs/hugetlb.go | 4 +-- libcontainer/cgroups/fs/hugetlb_test.go | 14 +++++----- libcontainer/cgroups/fs2/hugetlb.go | 4 +-- libcontainer/cgroups/utils.go | 30 +++++++++++++-------- libcontainer/cgroups/utils_test.go | 10 ------- 6 files changed, 45 insertions(+), 52 deletions(-) diff --git a/libcontainer/cgroups/fs/fs.go b/libcontainer/cgroups/fs/fs.go index 90a0b49b0..fb4fcc7f7 100644 --- a/libcontainer/cgroups/fs/fs.go +++ b/libcontainer/cgroups/fs/fs.go @@ -13,25 +13,22 @@ import ( "github.com/opencontainers/runc/libcontainer/configs" ) -var ( - subsystems = []subsystem{ - &CpusetGroup{}, - &DevicesGroup{}, - &MemoryGroup{}, - &CpuGroup{}, - &CpuacctGroup{}, - &PidsGroup{}, - &BlkioGroup{}, - &HugetlbGroup{}, - &NetClsGroup{}, - &NetPrioGroup{}, - &PerfEventGroup{}, - &FreezerGroup{}, - &RdmaGroup{}, - &NameGroup{GroupName: "name=systemd", Join: true}, - } - HugePageSizes = cgroups.HugePageSizes() -) +var subsystems = []subsystem{ + &CpusetGroup{}, + &DevicesGroup{}, + &MemoryGroup{}, + &CpuGroup{}, + &CpuacctGroup{}, + &PidsGroup{}, + &BlkioGroup{}, + &HugetlbGroup{}, + &NetClsGroup{}, + &NetPrioGroup{}, + &PerfEventGroup{}, + &FreezerGroup{}, + &RdmaGroup{}, + &NameGroup{GroupName: "name=systemd", Join: true}, +} var errSubsystemDoesNotExist = errors.New("cgroup: subsystem does not exist") diff --git a/libcontainer/cgroups/fs/hugetlb.go b/libcontainer/cgroups/fs/hugetlb.go index 86650128c..8ddd6fdd8 100644 --- a/libcontainer/cgroups/fs/hugetlb.go +++ b/libcontainer/cgroups/fs/hugetlb.go @@ -29,11 +29,11 @@ func (s *HugetlbGroup) Set(path string, r *configs.Resources) error { } func (s *HugetlbGroup) GetStats(path string, stats *cgroups.Stats) error { - hugetlbStats := cgroups.HugetlbStats{} if !cgroups.PathExists(path) { return nil } - for _, pageSize := range HugePageSizes { + hugetlbStats := cgroups.HugetlbStats{} + for _, pageSize := range cgroups.HugePageSizes() { usage := "hugetlb." + pageSize + ".usage_in_bytes" value, err := fscommon.GetCgroupParamUint(path, usage) if err != nil { diff --git a/libcontainer/cgroups/fs/hugetlb_test.go b/libcontainer/cgroups/fs/hugetlb_test.go index ba54f35a1..f4aea7eb5 100644 --- a/libcontainer/cgroups/fs/hugetlb_test.go +++ b/libcontainer/cgroups/fs/hugetlb_test.go @@ -31,14 +31,14 @@ func TestHugetlbSetHugetlb(t *testing.T) { hugetlbAfter = 512 ) - for _, pageSize := range HugePageSizes { + for _, pageSize := range cgroups.HugePageSizes() { writeFileContents(t, path, map[string]string{ fmt.Sprintf(limit, pageSize): strconv.Itoa(hugetlbBefore), }) } r := &configs.Resources{} - for _, pageSize := range HugePageSizes { + for _, pageSize := range cgroups.HugePageSizes() { r.HugetlbLimit = []*configs.HugepageLimit{ { Pagesize: pageSize, @@ -51,7 +51,7 @@ func TestHugetlbSetHugetlb(t *testing.T) { } } - for _, pageSize := range HugePageSizes { + for _, pageSize := range cgroups.HugePageSizes() { limit := fmt.Sprintf(limit, pageSize) value, err := fscommon.GetCgroupParamUint(path, limit) if err != nil { @@ -65,7 +65,7 @@ func TestHugetlbSetHugetlb(t *testing.T) { func TestHugetlbStats(t *testing.T) { path := tempDir(t, "hugetlb") - for _, pageSize := range HugePageSizes { + for _, pageSize := range cgroups.HugePageSizes() { writeFileContents(t, path, map[string]string{ fmt.Sprintf(usage, pageSize): hugetlbUsageContents, fmt.Sprintf(maxUsage, pageSize): hugetlbMaxUsageContents, @@ -80,7 +80,7 @@ func TestHugetlbStats(t *testing.T) { t.Fatal(err) } expectedStats := cgroups.HugetlbStats{Usage: 128, MaxUsage: 256, Failcnt: 100} - for _, pageSize := range HugePageSizes { + for _, pageSize := range cgroups.HugePageSizes() { expectHugetlbStatEquals(t, expectedStats, actualStats.HugetlbStats[pageSize]) } } @@ -101,7 +101,7 @@ func TestHugetlbStatsNoUsageFile(t *testing.T) { func TestHugetlbStatsNoMaxUsageFile(t *testing.T) { path := tempDir(t, "hugetlb") - for _, pageSize := range HugePageSizes { + for _, pageSize := range cgroups.HugePageSizes() { writeFileContents(t, path, map[string]string{ fmt.Sprintf(usage, pageSize): hugetlbUsageContents, }) @@ -117,7 +117,7 @@ func TestHugetlbStatsNoMaxUsageFile(t *testing.T) { func TestHugetlbStatsBadUsageFile(t *testing.T) { path := tempDir(t, "hugetlb") - for _, pageSize := range HugePageSizes { + for _, pageSize := range cgroups.HugePageSizes() { writeFileContents(t, path, map[string]string{ fmt.Sprintf(usage, pageSize): "bad", maxUsage: hugetlbMaxUsageContents, diff --git a/libcontainer/cgroups/fs2/hugetlb.go b/libcontainer/cgroups/fs2/hugetlb.go index 3955f0781..c92a7e64a 100644 --- a/libcontainer/cgroups/fs2/hugetlb.go +++ b/libcontainer/cgroups/fs2/hugetlb.go @@ -26,10 +26,8 @@ func setHugeTlb(dirPath string, r *configs.Resources) error { } func statHugeTlb(dirPath string, stats *cgroups.Stats) error { - hugePageSizes := cgroups.HugePageSizes() hugetlbStats := cgroups.HugetlbStats{} - - for _, pagesize := range hugePageSizes { + for _, pagesize := range cgroups.HugePageSizes() { value, err := fscommon.GetCgroupParamUint(dirPath, "hugetlb."+pagesize+".current") if err != nil { return err diff --git a/libcontainer/cgroups/utils.go b/libcontainer/cgroups/utils.go index f96e4dedf..1577c41ef 100644 --- a/libcontainer/cgroups/utils.go +++ b/libcontainer/cgroups/utils.go @@ -302,18 +302,26 @@ func RemovePaths(paths map[string]string) (err error) { return fmt.Errorf("Failed to remove paths: %v", paths) } -func HugePageSizes() []string { - dir, err := os.OpenFile("/sys/kernel/mm/hugepages", unix.O_DIRECTORY|unix.O_RDONLY, 0) - if err != nil { - return nil - } - files, err := dir.Readdirnames(0) - dir.Close() - if err != nil { - return nil - } +var ( + hugePageSizes []string + initHPSOnce sync.Once +) + +func HugePageSizes() []string { + initHPSOnce.Do(func() { + dir, err := os.OpenFile("/sys/kernel/mm/hugepages", unix.O_DIRECTORY|unix.O_RDONLY, 0) + if err != nil { + return + } + files, err := dir.Readdirnames(0) + dir.Close() + if err != nil { + return + } + + hugePageSizes, _ = getHugePageSizeFromFilenames(files) + }) - hugePageSizes, _ := getHugePageSizeFromFilenames(files) return hugePageSizes } diff --git a/libcontainer/cgroups/utils_test.go b/libcontainer/cgroups/utils_test.go index a0a261dee..708015d44 100644 --- a/libcontainer/cgroups/utils_test.go +++ b/libcontainer/cgroups/utils_test.go @@ -448,16 +448,6 @@ func TestFindCgroupMountpointAndRoot(t *testing.T) { } } -func BenchmarkHugePageSizes(b *testing.B) { - var output []string - for i := 0; i < b.N; i++ { - output = HugePageSizes() - } - if len(output) == 0 { - b.Fatal("unexpected results") - } -} - func BenchmarkGetHugePageSizeImpl(b *testing.B) { var ( input = []string{"hugepages-1048576kB", "hugepages-2048kB", "hugepages-32768kB", "hugepages-64kB"}