intelrdt: Add Cache Monitoring Technology stats

Signed-off-by: Paweł Szulik <pawel.szulik@intel.com>

events.go

@@ -161,9 +161,12 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
 			s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot
 			s.IntelRdt.MemBwSchema = is.MemBwSchema
 		}
-		if intelrdt.IsMbmEnabled() {
+		if intelrdt.IsMBMEnabled() {
 			s.IntelRdt.MBMStats = is.MBMStats
 		}
+		if intelrdt.IsCMTEnabled() {
+			s.IntelRdt.CMTStats = is.CMTStats
+		}
 	}
 
 	s.NetworkInterfaces = ls.Interfaces

libcontainer/intelrdt/cmt.go

@@ -0,0 +1,22 @@
+package intelrdt
+
+var (
+	cmtEnabled bool
+)
+
+// Check if Intel RDT/CMT is enabled.
+func IsCMTEnabled() bool {
+	return cmtEnabled
+}
+
+func getCMTNumaNodeStats(numaPath string) (*CMTNumaNodeStats, error) {
+	stats := &CMTNumaNodeStats{}
+
+	llcOccupancy, err := getIntelRdtParamUint(numaPath, "llc_occupancy")
+	if err != nil {
+		return nil, err
+	}
+	stats.LLCOccupancy = llcOccupancy
+
+	return stats, nil
+}

libcontainer/intelrdt/cmt_test.go

@@ -0,0 +1,56 @@
+package intelrdt
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestGetCMTNumaNodeStats(t *testing.T) {
+	mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"}
+
+	mocksFilesToCreate := map[string]uint64{
+		"llc_occupancy": 9123911,
+	}
+
+	mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate)
+
+	defer func() {
+		err := os.RemoveAll(mockedL3_MON)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Gather mbm", func(t *testing.T) {
+		enabledMonFeatures.llcOccupancy = true
+
+		stats := make([]CMTNumaNodeStats, 0, len(mocksNUMANodesToCreate))
+		for _, numa := range mocksNUMANodesToCreate {
+			other, err := getCMTNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa))
+			if err != nil {
+				t.Fatal(err)
+			}
+			stats = append(stats, *other)
+		}
+
+		expectedStats := CMTNumaNodeStats{
+			LLCOccupancy: mocksFilesToCreate["llc_occupancy"],
+		}
+
+		checkCMTStatCorrection(stats[0], expectedStats, t)
+		checkCMTStatCorrection(stats[1], expectedStats, t)
+	})
+}
+
+func checkCMTStatCorrection(got CMTNumaNodeStats, expected CMTNumaNodeStats, t *testing.T) {
+	if got.LLCOccupancy != expected.LLCOccupancy {
+		t.Fatalf("Wrong value of `llc_occupancy`. Expected: %v but got: %v",
+			expected.LLCOccupancy,
+			got.LLCOccupancy)
+	}
+}

libcontainer/intelrdt/intelrdt.go

@@ -228,7 +228,8 @@ func init() {
 
 	if flagsSet.MBMTotal || flagsSet.MBMLocal {
 		if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3_MON")); err == nil {
-			isMbmEnabled = true
+			mbmEnabled = true
+			cmtEnabled = true
 		}
 
 		enabledMonFeatures, err = getMonFeatures(intelRdtRoot)
@@ -665,12 +666,9 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) {
 		}
 	}
 
-	if IsMbmEnabled() {
-		mbmStats, err := getMBMStats(containerPath)
-		if err != nil {
-			return stats, err
-		}
-		stats.MBMStats = mbmStats
+	err = getMonitoringStats(containerPath, stats)
+	if err != nil {
+		return nil, err
 	}
 
 	return stats, nil

libcontainer/intelrdt/mbm.go

@@ -2,82 +2,14 @@
 
 package intelrdt
 
-import (
-	"bufio"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	"github.com/sirupsen/logrus"
-)
-
 var (
 	// The flag to indicate if Intel RDT/MBM is enabled
-	isMbmEnabled bool
-
-	enabledMonFeatures monFeatures
+	mbmEnabled bool
 )
 
-type monFeatures struct {
-	mbmTotalBytes bool
-	mbmLocalBytes bool
-}
-
-// Check if Intel RDT/MBM is enabled
-func IsMbmEnabled() bool {
-	return isMbmEnabled
-}
-
-func getMonFeatures(intelRdtRoot string) (monFeatures, error) {
-	file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features"))
-	defer file.Close()
-	if err != nil {
-		return monFeatures{}, err
-	}
-	return parseMonFeatures(file)
-}
-
-func parseMonFeatures(reader io.Reader) (monFeatures, error) {
-	scanner := bufio.NewScanner(reader)
-
-	monFeatures := monFeatures{}
-
-	for scanner.Scan() {
-
-		switch feature := scanner.Text(); feature {
-
-		case "mbm_total_bytes":
-			monFeatures.mbmTotalBytes = true
-		case "mbm_local_bytes":
-			monFeatures.mbmLocalBytes = true
-		default:
-			logrus.Warnf("Unsupported Intel RDT monitoring feature: %s", feature)
-		}
-	}
-
-	return monFeatures, scanner.Err()
-}
-
-func getMBMStats(containerPath string) (*[]MBMNumaNodeStats, error) {
-	var mbmStats []MBMNumaNodeStats
-
-	numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data"))
-	if err != nil {
-		return &mbmStats, err
-	}
-
-	for _, file := range numaFiles {
-		if file.IsDir() {
-			numaStats, err := getMBMNumaNodeStats(filepath.Join(containerPath, "mon_data", file.Name()))
-			if err != nil {
-				return &mbmStats, nil
-			}
-			mbmStats = append(mbmStats, *numaStats)
-		}
-	}
-
-	return &mbmStats, nil
+// Check if Intel RDT/MBM is enabled.
+func IsMBMEnabled() bool {
+	return mbmEnabled
 }
 
 func getMBMNumaNodeStats(numaPath string) (*MBMNumaNodeStats, error) {

libcontainer/intelrdt/mbm_test.go

@@ -3,71 +3,12 @@
 package intelrdt
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strconv"
-	"strings"
 	"testing"
 )
 
-func TestParseMonFeatures(t *testing.T) {
-	t.Run("All features available", func(t *testing.T) {
-		parsedMonFeatures, err := parseMonFeatures(
-			strings.NewReader("mbm_total_bytes\nmbm_local_bytes"))
-		if err != nil {
-			t.Errorf("Error while parsing mon features err = %v", err)
-		}
-
-		expectedMonFeatures := monFeatures{true, true}
-
-		if parsedMonFeatures != expectedMonFeatures {
-			t.Error("Cannot gather all features!")
-		}
-	})
-
-	t.Run("No features available", func(t *testing.T) {
-		parsedMonFeatures, err := parseMonFeatures(strings.NewReader(""))
-
-		if err != nil {
-			t.Errorf("Error while parsing mon features err = %v", err)
-		}
-
-		expectedMonFeatures := monFeatures{false, false}
-
-		if parsedMonFeatures != expectedMonFeatures {
-			t.Error("Expected no features available but there is any!")
-		}
-	})
-}
-
-func mockMBM(NUMANodes []string, mocks map[string]uint64) (string, error) {
-	testDir, err := ioutil.TempDir("", "rdt_mbm_test")
-	if err != nil {
-		return "", err
-	}
-	monDataPath := filepath.Join(testDir, "mon_data")
-
-	for _, numa := range NUMANodes {
-		numaPath := filepath.Join(monDataPath, numa)
-		err = os.MkdirAll(numaPath, os.ModePerm)
-		if err != nil {
-			return "", err
-		}
-
-		for fileName, value := range mocks {
-			err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777)
-			if err != nil {
-				return "", err
-			}
-		}
-
-	}
-
-	return testDir, nil
-}
-
-func TestGetMbmStats(t *testing.T) {
+func TestGetMBMNumaNodeStats(t *testing.T) {
 	mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"}
 
 	mocksFilesToCreate := map[string]uint64{
@@ -75,10 +16,10 @@ func TestGetMbmStats(t *testing.T) {
 		"mbm_local_bytes": 2361361,
 	}
 
-	mockedMBM, err := mockMBM(mocksNUMANodesToCreate, mocksFilesToCreate)
+	mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate)
 
 	defer func() {
-		err := os.RemoveAll(mockedMBM)
+		err := os.RemoveAll(mockedL3_MON)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -92,29 +33,13 @@ func TestGetMbmStats(t *testing.T) {
 		enabledMonFeatures.mbmTotalBytes = true
 		enabledMonFeatures.mbmLocalBytes = true
 
-		stats, err := getMBMStats(mockedMBM)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if len(*stats) != len(mocksNUMANodesToCreate) {
-			t.Fatalf("Wrong number of stats slices from NUMA nodes. Expected: %v but got: %v",
-				len(mocksNUMANodesToCreate), len(*stats))
-		}
-
-		checkStatCorrection := func(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) {
-			if got.MBMTotalBytes != expected.MBMTotalBytes {
-				t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v",
-					expected.MBMTotalBytes,
-					got.MBMTotalBytes)
+		stats := make([]MBMNumaNodeStats, 0, len(mocksNUMANodesToCreate))
+		for _, numa := range mocksNUMANodesToCreate {
+			other, err := getMBMNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa))
+			if err != nil {
+				t.Fatal(err)
 			}
-
-			if got.MBMLocalBytes != expected.MBMLocalBytes {
-				t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v",
-					expected.MBMLocalBytes,
-					got.MBMLocalBytes)
-			}
-
+			stats = append(stats, *other)
 		}
 
 		expectedStats := MBMNumaNodeStats{
@@ -122,7 +47,22 @@ func TestGetMbmStats(t *testing.T) {
 			MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"],
 		}
 
-		checkStatCorrection((*stats)[0], expectedStats, t)
-		checkStatCorrection((*stats)[1], expectedStats, t)
+		checkMBMStatCorrection(stats[0], expectedStats, t)
+		checkMBMStatCorrection(stats[1], expectedStats, t)
 	})
 }
+
+func checkMBMStatCorrection(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) {
+	if got.MBMTotalBytes != expected.MBMTotalBytes {
+		t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v",
+			expected.MBMTotalBytes,
+			got.MBMTotalBytes)
+	}
+
+	if got.MBMLocalBytes != expected.MBMLocalBytes {
+		t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v",
+			expected.MBMLocalBytes,
+			got.MBMLocalBytes)
+	}
+
+}

libcontainer/intelrdt/monitoring.go

@@ -0,0 +1,85 @@
+package intelrdt
+
+import (
+	"bufio"
+	"github.com/sirupsen/logrus"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+var (
+	enabledMonFeatures monFeatures
+)
+
+type monFeatures struct {
+	mbmTotalBytes bool
+	mbmLocalBytes bool
+	llcOccupancy  bool
+}
+
+func getMonFeatures(intelRdtRoot string) (monFeatures, error) {
+	file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features"))
+	defer file.Close()
+	if err != nil {
+		return monFeatures{}, err
+	}
+	return parseMonFeatures(file)
+}
+
+func parseMonFeatures(reader io.Reader) (monFeatures, error) {
+	scanner := bufio.NewScanner(reader)
+
+	monFeatures := monFeatures{}
+
+	for scanner.Scan() {
+		switch feature := scanner.Text(); feature {
+		case "mbm_total_bytes":
+			monFeatures.mbmTotalBytes = true
+		case "mbm_local_bytes":
+			monFeatures.mbmLocalBytes = true
+		case "llc_occupancy":
+			monFeatures.llcOccupancy = true
+		default:
+			logrus.Warnf("Unsupported Intel RDT monitoring feature: %s", feature)
+		}
+	}
+
+	return monFeatures, scanner.Err()
+}
+
+func getMonitoringStats(containerPath string, stats *Stats) error {
+	numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data"))
+	if err != nil {
+		return err
+	}
+
+	var mbmStats []MBMNumaNodeStats
+	var cmtStats []CMTNumaNodeStats
+
+	for _, file := range numaFiles {
+		if file.IsDir() {
+			numaPath := filepath.Join(containerPath, "mon_data", file.Name())
+			if IsMBMEnabled() {
+				numaMBMStats, err := getMBMNumaNodeStats(numaPath)
+				if err != nil {
+					return err
+				}
+				mbmStats = append(mbmStats, *numaMBMStats)
+			}
+			if IsCMTEnabled() {
+				numaCMTStats, err := getCMTNumaNodeStats(numaPath)
+				if err != nil {
+					return err
+				}
+				cmtStats = append(cmtStats, *numaCMTStats)
+			}
+		}
+	}
+
+	stats.MBMStats = &mbmStats
+	stats.CMTStats = &cmtStats
+
+	return err
+}

libcontainer/intelrdt/monitoring_test.go

@@ -0,0 +1,118 @@
+package intelrdt
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestParseMonFeatures(t *testing.T) {
+	t.Run("All features available", func(t *testing.T) {
+		parsedMonFeatures, err := parseMonFeatures(
+			strings.NewReader("mbm_total_bytes\nmbm_local_bytes\nllc_occupancy"))
+		if err != nil {
+			t.Errorf("Error while parsing mon features err = %v", err)
+		}
+
+		expectedMonFeatures := monFeatures{true, true, true}
+
+		if parsedMonFeatures != expectedMonFeatures {
+			t.Error("Cannot gather all features!")
+		}
+	})
+
+	t.Run("No features available", func(t *testing.T) {
+		parsedMonFeatures, err := parseMonFeatures(strings.NewReader(""))
+
+		if err != nil {
+			t.Errorf("Error while parsing mon features err = %v", err)
+		}
+
+		expectedMonFeatures := monFeatures{false, false, false}
+
+		if parsedMonFeatures != expectedMonFeatures {
+			t.Error("Expected no features available but there is any!")
+		}
+	})
+}
+
+func mockResctrlL3_MON(NUMANodes []string, mocks map[string]uint64) (string, error) {
+	testDir, err := ioutil.TempDir("", "rdt_mbm_test")
+	if err != nil {
+		return "", err
+	}
+	monDataPath := filepath.Join(testDir, "mon_data")
+
+	for _, numa := range NUMANodes {
+		numaPath := filepath.Join(monDataPath, numa)
+		err = os.MkdirAll(numaPath, os.ModePerm)
+		if err != nil {
+			return "", err
+		}
+
+		for fileName, value := range mocks {
+			err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777)
+			if err != nil {
+				return "", err
+			}
+		}
+
+	}
+
+	return testDir, nil
+}
+
+func TestGetMonitoringStats(t *testing.T) {
+	enabledMonFeatures.mbmTotalBytes = true
+	enabledMonFeatures.mbmLocalBytes = true
+	enabledMonFeatures.llcOccupancy = true
+	mbmEnabled = true
+	cmtEnabled = true
+
+	mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"}
+
+	mocksFilesToCreate := map[string]uint64{
+		"mbm_total_bytes": 9123911,
+		"mbm_local_bytes": 2361361,
+		"llc_occupancy":   123331,
+	}
+
+	mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate)
+
+	defer func() {
+		err := os.RemoveAll(mockedL3_MON)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Gather monitoring stats", func(t *testing.T) {
+		var stats Stats
+		err := getMonitoringStats(mockedL3_MON, &stats)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expectedMBMStats := MBMNumaNodeStats{
+			MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"],
+			MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"],
+		}
+
+		expectedCMTStats := CMTNumaNodeStats{LLCOccupancy: mocksFilesToCreate["llc_occupancy"]}
+
+		for _, gotMBMStat := range *stats.MBMStats {
+			checkMBMStatCorrection(gotMBMStat, expectedMBMStats, t)
+		}
+
+		for _, gotCMTStat := range *stats.CMTStats {
+			checkCMTStatCorrection(gotCMTStat, expectedCMTStats, t)
+		}
+	})
+}

libcontainer/intelrdt/stats.go

@@ -16,13 +16,18 @@ type MemBwInfo struct {
 }
 
 type MBMNumaNodeStats struct {
-	// The 'mbm_total_bytes' in 'container_id' group
+	// The 'mbm_total_bytes' in 'container_id' group.
 	MBMTotalBytes uint64 `json:"mbm_total_bytes,omitempty"`
 
-	// The 'mbm_local_bytes' in 'container_id' group
+	// The 'mbm_local_bytes' in 'container_id' group.
 	MBMLocalBytes uint64 `json:"mbm_local_bytes,omitempty"`
 }
 
+type CMTNumaNodeStats struct {
+	// The 'llc_occupancy' in 'container_id' group.
+	LLCOccupancy uint64 `json:"llc_occupancy,omitempty"`
+}
+
 type Stats struct {
 	// The read-only L3 cache information
 	L3CacheInfo *L3CacheInfo `json:"l3_cache_info,omitempty"`
@@ -43,7 +48,10 @@ type Stats struct {
 	MemBwSchema string `json:"mem_bw_schema,omitempty"`
 
 	// The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group
-	MBMStats *[]MBMNumaNodeStats `json:"mbm_statistics,omitempty"`
+	MBMStats *[]MBMNumaNodeStats `json:"mbm_stats,omitempty"`
+
+	// The cache monitoring technology statistics from NUMA nodes in 'container_id' group
+	CMTStats *[]CMTNumaNodeStats `json:"cmt_stats,omitempty"`
 }
 
 func NewStats() *Stats {

types/events.go

@@ -117,7 +117,10 @@ type IntelRdt struct {
 	MemBwSchema string `json:"mem_bw_schema,omitempty"`
 
 	// The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group
-	MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_statistics,omitempty"`
+	MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_stats,omitempty"`
+
+	// The cache monitoring technology statistics from NUMA nodes in 'container_id' group
+	CMTStats *[]intelrdt.CMTNumaNodeStats `json:"cmt_stats,omitempty"`
 }
 
 type NetworkInterface struct {