diff --git a/events.go b/events.go index b2d3087a3..fcda3b174 100644 --- a/events.go +++ b/events.go @@ -161,9 +161,12 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats { s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot s.IntelRdt.MemBwSchema = is.MemBwSchema } - if intelrdt.IsMbmEnabled() { + if intelrdt.IsMBMEnabled() { s.IntelRdt.MBMStats = is.MBMStats } + if intelrdt.IsCMTEnabled() { + s.IntelRdt.CMTStats = is.CMTStats + } } s.NetworkInterfaces = ls.Interfaces diff --git a/libcontainer/intelrdt/cmt.go b/libcontainer/intelrdt/cmt.go new file mode 100644 index 000000000..5c406e102 --- /dev/null +++ b/libcontainer/intelrdt/cmt.go @@ -0,0 +1,22 @@ +package intelrdt + +var ( + cmtEnabled bool +) + +// Check if Intel RDT/CMT is enabled. +func IsCMTEnabled() bool { + return cmtEnabled +} + +func getCMTNumaNodeStats(numaPath string) (*CMTNumaNodeStats, error) { + stats := &CMTNumaNodeStats{} + + llcOccupancy, err := getIntelRdtParamUint(numaPath, "llc_occupancy") + if err != nil { + return nil, err + } + stats.LLCOccupancy = llcOccupancy + + return stats, nil +} diff --git a/libcontainer/intelrdt/cmt_test.go b/libcontainer/intelrdt/cmt_test.go new file mode 100644 index 000000000..e061695e8 --- /dev/null +++ b/libcontainer/intelrdt/cmt_test.go @@ -0,0 +1,56 @@ +package intelrdt + +import ( + "os" + "path/filepath" + "testing" +) + +func TestGetCMTNumaNodeStats(t *testing.T) { + mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} + + mocksFilesToCreate := map[string]uint64{ + "llc_occupancy": 9123911, + } + + mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate) + + defer func() { + err := os.RemoveAll(mockedL3_MON) + if err != nil { + t.Fatal(err) + } + }() + + if err != nil { + t.Fatal(err) + } + + t.Run("Gather mbm", func(t *testing.T) { + enabledMonFeatures.llcOccupancy = true + + stats := make([]CMTNumaNodeStats, 0, len(mocksNUMANodesToCreate)) + for _, numa := range mocksNUMANodesToCreate { + other, err := getCMTNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa)) + if err != nil { + t.Fatal(err) + } + stats = append(stats, *other) + } + + expectedStats := CMTNumaNodeStats{ + LLCOccupancy: mocksFilesToCreate["llc_occupancy"], + } + + checkCMTStatCorrection(stats[0], expectedStats, t) + checkCMTStatCorrection(stats[1], expectedStats, t) + }) +} + +func checkCMTStatCorrection(got CMTNumaNodeStats, expected CMTNumaNodeStats, t *testing.T) { + if got.LLCOccupancy != expected.LLCOccupancy { + t.Fatalf("Wrong value of `llc_occupancy`. Expected: %v but got: %v", + expected.LLCOccupancy, + got.LLCOccupancy) + } +} diff --git a/libcontainer/intelrdt/intelrdt.go b/libcontainer/intelrdt/intelrdt.go index 6be7f8987..5b19d55a2 100644 --- a/libcontainer/intelrdt/intelrdt.go +++ b/libcontainer/intelrdt/intelrdt.go @@ -228,7 +228,8 @@ func init() { if flagsSet.MBMTotal || flagsSet.MBMLocal { if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3_MON")); err == nil { - isMbmEnabled = true + mbmEnabled = true + cmtEnabled = true } enabledMonFeatures, err = getMonFeatures(intelRdtRoot) @@ -665,12 +666,9 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) { } } - if IsMbmEnabled() { - mbmStats, err := getMBMStats(containerPath) - if err != nil { - return stats, err - } - stats.MBMStats = mbmStats + err = getMonitoringStats(containerPath, stats) + if err != nil { + return nil, err } return stats, nil diff --git a/libcontainer/intelrdt/mbm.go b/libcontainer/intelrdt/mbm.go index c76f51559..3730bab56 100644 --- a/libcontainer/intelrdt/mbm.go +++ b/libcontainer/intelrdt/mbm.go @@ -2,82 +2,14 @@ package intelrdt -import ( - "bufio" - "io" - "io/ioutil" - "os" - "path/filepath" - - "github.com/sirupsen/logrus" -) - var ( // The flag to indicate if Intel RDT/MBM is enabled - isMbmEnabled bool - - enabledMonFeatures monFeatures + mbmEnabled bool ) -type monFeatures struct { - mbmTotalBytes bool - mbmLocalBytes bool -} - -// Check if Intel RDT/MBM is enabled -func IsMbmEnabled() bool { - return isMbmEnabled -} - -func getMonFeatures(intelRdtRoot string) (monFeatures, error) { - file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features")) - defer file.Close() - if err != nil { - return monFeatures{}, err - } - return parseMonFeatures(file) -} - -func parseMonFeatures(reader io.Reader) (monFeatures, error) { - scanner := bufio.NewScanner(reader) - - monFeatures := monFeatures{} - - for scanner.Scan() { - - switch feature := scanner.Text(); feature { - - case "mbm_total_bytes": - monFeatures.mbmTotalBytes = true - case "mbm_local_bytes": - monFeatures.mbmLocalBytes = true - default: - logrus.Warnf("Unsupported Intel RDT monitoring feature: %s", feature) - } - } - - return monFeatures, scanner.Err() -} - -func getMBMStats(containerPath string) (*[]MBMNumaNodeStats, error) { - var mbmStats []MBMNumaNodeStats - - numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data")) - if err != nil { - return &mbmStats, err - } - - for _, file := range numaFiles { - if file.IsDir() { - numaStats, err := getMBMNumaNodeStats(filepath.Join(containerPath, "mon_data", file.Name())) - if err != nil { - return &mbmStats, nil - } - mbmStats = append(mbmStats, *numaStats) - } - } - - return &mbmStats, nil +// Check if Intel RDT/MBM is enabled. +func IsMBMEnabled() bool { + return mbmEnabled } func getMBMNumaNodeStats(numaPath string) (*MBMNumaNodeStats, error) { diff --git a/libcontainer/intelrdt/mbm_test.go b/libcontainer/intelrdt/mbm_test.go index 9ddf8ba21..ae0f202cd 100644 --- a/libcontainer/intelrdt/mbm_test.go +++ b/libcontainer/intelrdt/mbm_test.go @@ -3,71 +3,12 @@ package intelrdt import ( - "io/ioutil" "os" "path/filepath" - "strconv" - "strings" "testing" ) -func TestParseMonFeatures(t *testing.T) { - t.Run("All features available", func(t *testing.T) { - parsedMonFeatures, err := parseMonFeatures( - strings.NewReader("mbm_total_bytes\nmbm_local_bytes")) - if err != nil { - t.Errorf("Error while parsing mon features err = %v", err) - } - - expectedMonFeatures := monFeatures{true, true} - - if parsedMonFeatures != expectedMonFeatures { - t.Error("Cannot gather all features!") - } - }) - - t.Run("No features available", func(t *testing.T) { - parsedMonFeatures, err := parseMonFeatures(strings.NewReader("")) - - if err != nil { - t.Errorf("Error while parsing mon features err = %v", err) - } - - expectedMonFeatures := monFeatures{false, false} - - if parsedMonFeatures != expectedMonFeatures { - t.Error("Expected no features available but there is any!") - } - }) -} - -func mockMBM(NUMANodes []string, mocks map[string]uint64) (string, error) { - testDir, err := ioutil.TempDir("", "rdt_mbm_test") - if err != nil { - return "", err - } - monDataPath := filepath.Join(testDir, "mon_data") - - for _, numa := range NUMANodes { - numaPath := filepath.Join(monDataPath, numa) - err = os.MkdirAll(numaPath, os.ModePerm) - if err != nil { - return "", err - } - - for fileName, value := range mocks { - err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777) - if err != nil { - return "", err - } - } - - } - - return testDir, nil -} - -func TestGetMbmStats(t *testing.T) { +func TestGetMBMNumaNodeStats(t *testing.T) { mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} mocksFilesToCreate := map[string]uint64{ @@ -75,10 +16,10 @@ func TestGetMbmStats(t *testing.T) { "mbm_local_bytes": 2361361, } - mockedMBM, err := mockMBM(mocksNUMANodesToCreate, mocksFilesToCreate) + mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate) defer func() { - err := os.RemoveAll(mockedMBM) + err := os.RemoveAll(mockedL3_MON) if err != nil { t.Fatal(err) } @@ -92,29 +33,13 @@ func TestGetMbmStats(t *testing.T) { enabledMonFeatures.mbmTotalBytes = true enabledMonFeatures.mbmLocalBytes = true - stats, err := getMBMStats(mockedMBM) - if err != nil { - t.Fatal(err) - } - - if len(*stats) != len(mocksNUMANodesToCreate) { - t.Fatalf("Wrong number of stats slices from NUMA nodes. Expected: %v but got: %v", - len(mocksNUMANodesToCreate), len(*stats)) - } - - checkStatCorrection := func(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) { - if got.MBMTotalBytes != expected.MBMTotalBytes { - t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v", - expected.MBMTotalBytes, - got.MBMTotalBytes) + stats := make([]MBMNumaNodeStats, 0, len(mocksNUMANodesToCreate)) + for _, numa := range mocksNUMANodesToCreate { + other, err := getMBMNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa)) + if err != nil { + t.Fatal(err) } - - if got.MBMLocalBytes != expected.MBMLocalBytes { - t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v", - expected.MBMLocalBytes, - got.MBMLocalBytes) - } - + stats = append(stats, *other) } expectedStats := MBMNumaNodeStats{ @@ -122,7 +47,22 @@ func TestGetMbmStats(t *testing.T) { MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"], } - checkStatCorrection((*stats)[0], expectedStats, t) - checkStatCorrection((*stats)[1], expectedStats, t) + checkMBMStatCorrection(stats[0], expectedStats, t) + checkMBMStatCorrection(stats[1], expectedStats, t) }) } + +func checkMBMStatCorrection(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) { + if got.MBMTotalBytes != expected.MBMTotalBytes { + t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v", + expected.MBMTotalBytes, + got.MBMTotalBytes) + } + + if got.MBMLocalBytes != expected.MBMLocalBytes { + t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v", + expected.MBMLocalBytes, + got.MBMLocalBytes) + } + +} diff --git a/libcontainer/intelrdt/monitoring.go b/libcontainer/intelrdt/monitoring.go new file mode 100644 index 000000000..4ccc8eae0 --- /dev/null +++ b/libcontainer/intelrdt/monitoring.go @@ -0,0 +1,85 @@ +package intelrdt + +import ( + "bufio" + "github.com/sirupsen/logrus" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +var ( + enabledMonFeatures monFeatures +) + +type monFeatures struct { + mbmTotalBytes bool + mbmLocalBytes bool + llcOccupancy bool +} + +func getMonFeatures(intelRdtRoot string) (monFeatures, error) { + file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features")) + defer file.Close() + if err != nil { + return monFeatures{}, err + } + return parseMonFeatures(file) +} + +func parseMonFeatures(reader io.Reader) (monFeatures, error) { + scanner := bufio.NewScanner(reader) + + monFeatures := monFeatures{} + + for scanner.Scan() { + switch feature := scanner.Text(); feature { + case "mbm_total_bytes": + monFeatures.mbmTotalBytes = true + case "mbm_local_bytes": + monFeatures.mbmLocalBytes = true + case "llc_occupancy": + monFeatures.llcOccupancy = true + default: + logrus.Warnf("Unsupported Intel RDT monitoring feature: %s", feature) + } + } + + return monFeatures, scanner.Err() +} + +func getMonitoringStats(containerPath string, stats *Stats) error { + numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data")) + if err != nil { + return err + } + + var mbmStats []MBMNumaNodeStats + var cmtStats []CMTNumaNodeStats + + for _, file := range numaFiles { + if file.IsDir() { + numaPath := filepath.Join(containerPath, "mon_data", file.Name()) + if IsMBMEnabled() { + numaMBMStats, err := getMBMNumaNodeStats(numaPath) + if err != nil { + return err + } + mbmStats = append(mbmStats, *numaMBMStats) + } + if IsCMTEnabled() { + numaCMTStats, err := getCMTNumaNodeStats(numaPath) + if err != nil { + return err + } + cmtStats = append(cmtStats, *numaCMTStats) + } + } + } + + stats.MBMStats = &mbmStats + stats.CMTStats = &cmtStats + + return err +} diff --git a/libcontainer/intelrdt/monitoring_test.go b/libcontainer/intelrdt/monitoring_test.go new file mode 100644 index 000000000..ec886a7ee --- /dev/null +++ b/libcontainer/intelrdt/monitoring_test.go @@ -0,0 +1,118 @@ +package intelrdt + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestParseMonFeatures(t *testing.T) { + t.Run("All features available", func(t *testing.T) { + parsedMonFeatures, err := parseMonFeatures( + strings.NewReader("mbm_total_bytes\nmbm_local_bytes\nllc_occupancy")) + if err != nil { + t.Errorf("Error while parsing mon features err = %v", err) + } + + expectedMonFeatures := monFeatures{true, true, true} + + if parsedMonFeatures != expectedMonFeatures { + t.Error("Cannot gather all features!") + } + }) + + t.Run("No features available", func(t *testing.T) { + parsedMonFeatures, err := parseMonFeatures(strings.NewReader("")) + + if err != nil { + t.Errorf("Error while parsing mon features err = %v", err) + } + + expectedMonFeatures := monFeatures{false, false, false} + + if parsedMonFeatures != expectedMonFeatures { + t.Error("Expected no features available but there is any!") + } + }) +} + +func mockResctrlL3_MON(NUMANodes []string, mocks map[string]uint64) (string, error) { + testDir, err := ioutil.TempDir("", "rdt_mbm_test") + if err != nil { + return "", err + } + monDataPath := filepath.Join(testDir, "mon_data") + + for _, numa := range NUMANodes { + numaPath := filepath.Join(monDataPath, numa) + err = os.MkdirAll(numaPath, os.ModePerm) + if err != nil { + return "", err + } + + for fileName, value := range mocks { + err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777) + if err != nil { + return "", err + } + } + + } + + return testDir, nil +} + +func TestGetMonitoringStats(t *testing.T) { + enabledMonFeatures.mbmTotalBytes = true + enabledMonFeatures.mbmLocalBytes = true + enabledMonFeatures.llcOccupancy = true + mbmEnabled = true + cmtEnabled = true + + mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} + + mocksFilesToCreate := map[string]uint64{ + "mbm_total_bytes": 9123911, + "mbm_local_bytes": 2361361, + "llc_occupancy": 123331, + } + + mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate) + + defer func() { + err := os.RemoveAll(mockedL3_MON) + if err != nil { + t.Fatal(err) + } + }() + + if err != nil { + t.Fatal(err) + } + + t.Run("Gather monitoring stats", func(t *testing.T) { + var stats Stats + err := getMonitoringStats(mockedL3_MON, &stats) + if err != nil { + t.Fatal(err) + } + + expectedMBMStats := MBMNumaNodeStats{ + MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"], + MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"], + } + + expectedCMTStats := CMTNumaNodeStats{LLCOccupancy: mocksFilesToCreate["llc_occupancy"]} + + for _, gotMBMStat := range *stats.MBMStats { + checkMBMStatCorrection(gotMBMStat, expectedMBMStats, t) + } + + for _, gotCMTStat := range *stats.CMTStats { + checkCMTStatCorrection(gotCMTStat, expectedCMTStats, t) + } + }) +} diff --git a/libcontainer/intelrdt/stats.go b/libcontainer/intelrdt/stats.go index f90293be0..eeb0ee9f0 100644 --- a/libcontainer/intelrdt/stats.go +++ b/libcontainer/intelrdt/stats.go @@ -16,13 +16,18 @@ type MemBwInfo struct { } type MBMNumaNodeStats struct { - // The 'mbm_total_bytes' in 'container_id' group + // The 'mbm_total_bytes' in 'container_id' group. MBMTotalBytes uint64 `json:"mbm_total_bytes,omitempty"` - // The 'mbm_local_bytes' in 'container_id' group + // The 'mbm_local_bytes' in 'container_id' group. MBMLocalBytes uint64 `json:"mbm_local_bytes,omitempty"` } +type CMTNumaNodeStats struct { + // The 'llc_occupancy' in 'container_id' group. + LLCOccupancy uint64 `json:"llc_occupancy,omitempty"` +} + type Stats struct { // The read-only L3 cache information L3CacheInfo *L3CacheInfo `json:"l3_cache_info,omitempty"` @@ -43,7 +48,10 @@ type Stats struct { MemBwSchema string `json:"mem_bw_schema,omitempty"` // The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group - MBMStats *[]MBMNumaNodeStats `json:"mbm_statistics,omitempty"` + MBMStats *[]MBMNumaNodeStats `json:"mbm_stats,omitempty"` + + // The cache monitoring technology statistics from NUMA nodes in 'container_id' group + CMTStats *[]CMTNumaNodeStats `json:"cmt_stats,omitempty"` } func NewStats() *Stats { diff --git a/types/events.go b/types/events.go index bdc0a5151..34d170c8d 100644 --- a/types/events.go +++ b/types/events.go @@ -117,7 +117,10 @@ type IntelRdt struct { MemBwSchema string `json:"mem_bw_schema,omitempty"` // The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group - MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_statistics,omitempty"` + MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_stats,omitempty"` + + // The cache monitoring technology statistics from NUMA nodes in 'container_id' group + CMTStats *[]intelrdt.CMTNumaNodeStats `json:"cmt_stats,omitempty"` } type NetworkInterface struct {