From 990a6c57f1e23343400cdcaaf44ed654421a212d Mon Sep 17 00:00:00 2001 From: Piotr Wagner Date: Fri, 31 Jul 2020 08:31:39 +0200 Subject: [PATCH 1/2] cgroups: update blkio GetStats Signed-off-by: Piotr Wagner --- libcontainer/cgroups/fs/blkio.go | 124 +++++- libcontainer/cgroups/fs/blkio_test.go | 524 +++++++++++++++++--------- 2 files changed, 449 insertions(+), 199 deletions(-) diff --git a/libcontainer/cgroups/fs/blkio.go b/libcontainer/cgroups/fs/blkio.go index 4d5e5beb4..593a4b945 100644 --- a/libcontainer/cgroups/fs/blkio.go +++ b/libcontainer/cgroups/fs/blkio.go @@ -157,61 +157,153 @@ func getBlkioStat(dir, file string) ([]cgroups.BlkioStatEntry, error) { } func (s *BlkioGroup) GetStats(path string, stats *cgroups.Stats) error { - // Try to read CFQ stats available on all CFQ enabled kernels first - if blkioStats, err := getBlkioStat(path, "blkio.io_serviced_recursive"); err == nil && blkioStats != nil { - return getCFQStats(path, stats) + // Try to read BFQ stats with debug files enabled first + if moveToNextMethod, err := getBFQDebugStats(path, stats); !moveToNextMethod { + return err } - return getStats(path, stats) // Use generic stats as fallback + // Try to read BFQ stats with debug files disabled + if moveToNextMethod, err := getBFQStats(path, stats); !moveToNextMethod { + return err + } + // Try to read CFQ stats available on all CFQ enabled kernels + if moveToNextMethod, err := getCFQStats(path, stats); !moveToNextMethod { + return err + } + // Try to read recursive version of generic stats + if moveToNextMethod, err := getRecursiveThrottleStats(path, stats); !moveToNextMethod { + return err + } + return getBaseStats(path, stats) // Use generic stats as fallback } -func getCFQStats(path string, stats *cgroups.Stats) error { +func getBFQStats(path string, stats *cgroups.Stats) (bool, error) { var blkioStats []cgroups.BlkioStatEntry var err error + var moveToNextMethod = false - if blkioStats, err = getBlkioStat(path, "blkio.sectors_recursive"); err != nil { - return err + if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_serviced_recursive"); err != nil || blkioStats == nil { + moveToNextMethod = true + return moveToNextMethod, err + } + stats.BlkioStats.IoServicedRecursive = blkioStats + + if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_service_bytes_recursive"); err != nil { + return moveToNextMethod, err + } + stats.BlkioStats.IoServiceBytesRecursive = blkioStats + + return moveToNextMethod, nil +} + +func getBFQDebugStats(path string, stats *cgroups.Stats) (bool, error) { + var blkioStats []cgroups.BlkioStatEntry + var err error + var moveToNextMethod = false + + if blkioStats, err = getBlkioStat(path, "blkio.bfq.sectors_recursive"); err != nil || blkioStats == nil { + moveToNextMethod = true + return moveToNextMethod, err + } + stats.BlkioStats.SectorsRecursive = blkioStats + + if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_service_time_recursive"); err != nil { + return moveToNextMethod, err + } + stats.BlkioStats.IoServiceTimeRecursive = blkioStats + + if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_wait_time_recursive"); err != nil { + return moveToNextMethod, err + } + stats.BlkioStats.IoWaitTimeRecursive = blkioStats + + if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_merged_recursive"); err != nil { + return moveToNextMethod, err + } + stats.BlkioStats.IoMergedRecursive = blkioStats + + if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_queued_recursive"); err != nil { + return moveToNextMethod, err + } + stats.BlkioStats.IoQueuedRecursive = blkioStats + + if blkioStats, err = getBlkioStat(path, "blkio.bfq.time_recursive"); err != nil { + return moveToNextMethod, err + } + stats.BlkioStats.IoTimeRecursive = blkioStats + + _, err = getBFQStats(path, stats) + return moveToNextMethod, err +} + +func getCFQStats(path string, stats *cgroups.Stats) (bool, error) { + var blkioStats []cgroups.BlkioStatEntry + var err error + var moveToNextMethod = false + + if blkioStats, err = getBlkioStat(path, "blkio.sectors_recursive"); err != nil || blkioStats == nil { + moveToNextMethod = true + return moveToNextMethod, err } stats.BlkioStats.SectorsRecursive = blkioStats if blkioStats, err = getBlkioStat(path, "blkio.io_service_bytes_recursive"); err != nil { - return err + return moveToNextMethod, err } stats.BlkioStats.IoServiceBytesRecursive = blkioStats if blkioStats, err = getBlkioStat(path, "blkio.io_serviced_recursive"); err != nil { - return err + return moveToNextMethod, err } stats.BlkioStats.IoServicedRecursive = blkioStats if blkioStats, err = getBlkioStat(path, "blkio.io_queued_recursive"); err != nil { - return err + return moveToNextMethod, err } stats.BlkioStats.IoQueuedRecursive = blkioStats if blkioStats, err = getBlkioStat(path, "blkio.io_service_time_recursive"); err != nil { - return err + return moveToNextMethod, err } stats.BlkioStats.IoServiceTimeRecursive = blkioStats if blkioStats, err = getBlkioStat(path, "blkio.io_wait_time_recursive"); err != nil { - return err + return moveToNextMethod, err } stats.BlkioStats.IoWaitTimeRecursive = blkioStats if blkioStats, err = getBlkioStat(path, "blkio.io_merged_recursive"); err != nil { - return err + return moveToNextMethod, err } stats.BlkioStats.IoMergedRecursive = blkioStats if blkioStats, err = getBlkioStat(path, "blkio.time_recursive"); err != nil { - return err + return moveToNextMethod, err } stats.BlkioStats.IoTimeRecursive = blkioStats - return nil + return moveToNextMethod, nil } -func getStats(path string, stats *cgroups.Stats) error { +func getRecursiveThrottleStats(path string, stats *cgroups.Stats) (bool, error) { + var blkioStats []cgroups.BlkioStatEntry + var err error + var moveToNextMethod = false + + if blkioStats, err = getBlkioStat(path, "blkio.throttle.io_service_bytes_recursive"); err != nil || blkioStats == nil { + moveToNextMethod = true + return moveToNextMethod, err + } + stats.BlkioStats.IoServiceBytesRecursive = blkioStats + + if blkioStats, err = getBlkioStat(path, "blkio.throttle.io_serviced_recursive"); err != nil { + return moveToNextMethod, err + } + stats.BlkioStats.IoServicedRecursive = blkioStats + + return moveToNextMethod, nil +} + +func getBaseStats(path string, stats *cgroups.Stats) error { var blkioStats []cgroups.BlkioStatEntry var err error diff --git a/libcontainer/cgroups/fs/blkio_test.go b/libcontainer/cgroups/fs/blkio_test.go index 5ba60fa7f..d26b9d23c 100644 --- a/libcontainer/cgroups/fs/blkio_test.go +++ b/libcontainer/cgroups/fs/blkio_test.go @@ -3,6 +3,7 @@ package fs import ( + "fmt" "strconv" "testing" @@ -13,43 +14,81 @@ import ( const ( sectorsRecursiveContents = `8:0 1024` + sectorsRecursiveContentsBFQ = `8:0 2048` serviceBytesRecursiveContents = `8:0 Read 100 8:0 Write 200 8:0 Sync 300 8:0 Async 500 8:0 Total 500 Total 500` + + serviceBytesRecursiveContentsBFQ = `8:0 Read 1100 +8:0 Write 1200 +8:0 Sync 1300 +8:0 Async 1500 +8:0 Total 1500 +Total 1500` servicedRecursiveContents = `8:0 Read 10 8:0 Write 40 8:0 Sync 20 8:0 Async 30 8:0 Total 50 Total 50` + servicedRecursiveContentsBFQ = `8:0 Read 11 +8:0 Write 41 +8:0 Sync 21 +8:0 Async 31 +8:0 Total 51 +Total 51` queuedRecursiveContents = `8:0 Read 1 8:0 Write 4 8:0 Sync 2 8:0 Async 3 8:0 Total 5 Total 5` + queuedRecursiveContentsBFQ = `8:0 Read 2 +8:0 Write 3 +8:0 Sync 4 +8:0 Async 5 +8:0 Total 6 +Total 6` serviceTimeRecursiveContents = `8:0 Read 173959 8:0 Write 0 8:0 Sync 0 8:0 Async 173959 8:0 Total 17395 Total 17395` + serviceTimeRecursiveContentsBFQ = `8:0 Read 173959 +8:0 Write 0 +8:0 Sync 0 +8:0 Async 173 +8:0 Total 174 +Total 174` waitTimeRecursiveContents = `8:0 Read 15571 8:0 Write 0 8:0 Sync 0 8:0 Async 15571 8:0 Total 15571` + waitTimeRecursiveContentsBFQ = `8:0 Read 1557 +8:0 Write 0 +8:0 Sync 0 +8:0 Async 1557 +8:0 Total 1557` mergedRecursiveContents = `8:0 Read 5 8:0 Write 10 8:0 Sync 0 8:0 Async 0 8:0 Total 15 Total 15` - timeRecursiveContents = `8:0 8` - throttleServiceBytes = `8:0 Read 11030528 + mergedRecursiveContentsBFQ = `8:0 Read 51 +8:0 Write 101 +8:0 Sync 0 +8:0 Async 0 +8:0 Total 151 +Total 151` + timeRecursiveContents = `8:0 8` + timeRecursiveContentsBFQ = `8:0 16` + throttleServiceBytes = `8:0 Read 11030528 8:0 Write 23 8:0 Sync 42 8:0 Async 11030528 @@ -60,6 +99,17 @@ Total 15` 252:0 Async 11030528 252:0 Total 11030528 Total 22061056` + throttleServiceBytesRecursive = `8:0 Read 110305281 +8:0 Write 231 +8:0 Sync 421 +8:0 Async 110305281 +8:0 Total 110305281 +252:0 Read 110305281 +252:0 Write 231 +252:0 Sync 421 +252:0 Async 110305281 +252:0 Total 110305281 +Total 220610561` throttleServiced = `8:0 Read 164 8:0 Write 23 8:0 Sync 42 @@ -71,8 +121,51 @@ Total 22061056` 252:0 Async 164 252:0 Total 164 Total 328` + throttleServicedRecursive = `8:0 Read 1641 +8:0 Write 231 +8:0 Sync 421 +8:0 Async 1641 +8:0 Total 1641 +252:0 Read 1641 +252:0 Write 231 +252:0 Sync 421 +252:0 Async 1641 +252:0 Total 1641 +Total 3281` ) +var blkioBFQDebugStatsTestFiles = map[string]string{ + "blkio.bfq.io_service_bytes_recursive": serviceBytesRecursiveContentsBFQ, + "blkio.bfq.io_serviced_recursive": servicedRecursiveContentsBFQ, + "blkio.bfq.io_queued_recursive": queuedRecursiveContentsBFQ, + "blkio.bfq.io_service_time_recursive": serviceTimeRecursiveContentsBFQ, + "blkio.bfq.io_wait_time_recursive": waitTimeRecursiveContentsBFQ, + "blkio.bfq.io_merged_recursive": mergedRecursiveContentsBFQ, + "blkio.bfq.time_recursive": timeRecursiveContentsBFQ, + "blkio.bfq.sectors_recursive": sectorsRecursiveContentsBFQ, +} + +var blkioBFQStatsTestFiles = map[string]string{ + "blkio.bfq.io_service_bytes_recursive": serviceBytesRecursiveContentsBFQ, + "blkio.bfq.io_serviced_recursive": servicedRecursiveContentsBFQ, +} + +var blkioCFQStatsTestFiles = map[string]string{ + "blkio.io_service_bytes_recursive": serviceBytesRecursiveContents, + "blkio.io_serviced_recursive": servicedRecursiveContents, + "blkio.io_queued_recursive": queuedRecursiveContents, + "blkio.io_service_time_recursive": serviceTimeRecursiveContents, + "blkio.io_wait_time_recursive": waitTimeRecursiveContents, + "blkio.io_merged_recursive": mergedRecursiveContents, + "blkio.time_recursive": timeRecursiveContents, + "blkio.sectors_recursive": sectorsRecursiveContents, +} + +type blkioStatFailureTestCase struct { + desc string + filename string +} + func appendBlkioStatEntry(blkioStatEntries *[]cgroups.BlkioStatEntry, major, minor, value uint64, op string) { *blkioStatEntries = append(*blkioStatEntries, cgroups.BlkioStatEntry{Major: major, Minor: minor, Value: value, Op: op}) } @@ -174,19 +267,153 @@ func TestBlkioSetMultipleWeightDevice(t *testing.T) { } } -func TestBlkioStats(t *testing.T) { +func TestBlkioBFQDebugStats(t *testing.T) { helper := NewCgroupTestUtil("blkio", t) defer helper.cleanup() - helper.writeFileContents(map[string]string{ - "blkio.io_service_bytes_recursive": serviceBytesRecursiveContents, - "blkio.io_serviced_recursive": servicedRecursiveContents, - "blkio.io_queued_recursive": queuedRecursiveContents, - "blkio.io_service_time_recursive": serviceTimeRecursiveContents, - "blkio.io_wait_time_recursive": waitTimeRecursiveContents, - "blkio.io_merged_recursive": mergedRecursiveContents, - "blkio.time_recursive": timeRecursiveContents, - "blkio.sectors_recursive": sectorsRecursiveContents, - }) + helper.writeFileContents(blkioBFQDebugStatsTestFiles) + blkio := &BlkioGroup{} + actualStats := *cgroups.NewStats() + err := blkio.GetStats(helper.CgroupPath, &actualStats) + if err != nil { + t.Fatal(err) + } + + expectedStats := cgroups.BlkioStats{} + appendBlkioStatEntry(&expectedStats.SectorsRecursive, 8, 0, 2048, "") + + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1100, "Read") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1200, "Write") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1300, "Sync") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Async") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Total") + + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 11, "Read") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 41, "Write") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 21, "Sync") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 31, "Async") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 51, "Total") + + appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 2, "Read") + appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 3, "Write") + appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 4, "Sync") + appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 5, "Async") + appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 6, "Total") + + appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Read") + appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Write") + appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Sync") + appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173, "Async") + appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 174, "Total") + + appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Read") + appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Write") + appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Sync") + appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Async") + appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Total") + + appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 51, "Read") + appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 101, "Write") + appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Sync") + appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Async") + appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 151, "Total") + + appendBlkioStatEntry(&expectedStats.IoTimeRecursive, 8, 0, 16, "") + + expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats) +} + +func TestBlkioBFQStats(t *testing.T) { + helper := NewCgroupTestUtil("blkio", t) + defer helper.cleanup() + helper.writeFileContents(blkioBFQStatsTestFiles) + blkio := &BlkioGroup{} + actualStats := *cgroups.NewStats() + err := blkio.GetStats(helper.CgroupPath, &actualStats) + if err != nil { + t.Fatal(err) + } + + expectedStats := cgroups.BlkioStats{} + + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1100, "Read") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1200, "Write") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1300, "Sync") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Async") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Total") + + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 11, "Read") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 41, "Write") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 21, "Sync") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 31, "Async") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 51, "Total") + + expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats) +} + +func TestBlkioStatsNoFilesBFQDebug(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + testCases := []blkioStatFailureTestCase{ + { + desc: "missing blkio.bfq.io_service_bytes_recursive file", + filename: "blkio.bfq.io_service_bytes_recursive", + }, + { + desc: "missing blkio.bfq.io_serviced_recursive file", + filename: "blkio.bfq.io_serviced_recursive", + }, + { + desc: "missing blkio.bfq.io_queued_recursive file", + filename: "blkio.bfq.io_queued_recursive", + }, + { + desc: "missing blkio.bfq.sectors_recursive file", + filename: "blkio.bfq.sectors_recursive", + }, + { + desc: "missing blkio.bfq.io_service_time_recursive file", + filename: "blkio.bfq.io_service_time_recursive", + }, + { + desc: "missing blkio.bfq.io_wait_time_recursive file", + filename: "blkio.bfq.io_wait_time_recursive", + }, + { + desc: "missing blkio.bfq.io_merged_recursive file", + filename: "blkio.bfq.io_merged_recursive", + }, + { + desc: "missing blkio.bfq.time_recursive file", + filename: "blkio.bfq.time_recursive", + }, + } + + for _, testCase := range testCases { + helper := NewCgroupTestUtil("cpuset", t) + defer helper.cleanup() + + tempBlkioTestFiles := map[string]string{} + for i, v := range blkioBFQDebugStatsTestFiles { + tempBlkioTestFiles[i] = v + } + delete(tempBlkioTestFiles, testCase.filename) + + helper.writeFileContents(tempBlkioTestFiles) + cpuset := &CpusetGroup{} + actualStats := *cgroups.NewStats() + err := cpuset.GetStats(helper.CgroupPath, &actualStats) + + if err != nil { + t.Errorf(fmt.Sprintf("test case '%s' failed unexpectedly: %s", testCase.desc, err)) + } + } +} + +func TestBlkioCFQStats(t *testing.T) { + helper := NewCgroupTestUtil("blkio", t) + defer helper.cleanup() + helper.writeFileContents(blkioCFQStatsTestFiles) blkio := &BlkioGroup{} actualStats := *cgroups.NewStats() @@ -240,183 +467,63 @@ func TestBlkioStats(t *testing.T) { expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats) } -func TestBlkioStatsNoSectorsFile(t *testing.T) { - helper := NewCgroupTestUtil("blkio", t) - defer helper.cleanup() - helper.writeFileContents(map[string]string{ - "blkio.io_service_bytes_recursive": serviceBytesRecursiveContents, - "blkio.io_serviced_recursive": servicedRecursiveContents, - "blkio.io_queued_recursive": queuedRecursiveContents, - "blkio.io_service_time_recursive": serviceTimeRecursiveContents, - "blkio.io_wait_time_recursive": waitTimeRecursiveContents, - "blkio.io_merged_recursive": mergedRecursiveContents, - "blkio.time_recursive": timeRecursiveContents, - }) - - blkio := &BlkioGroup{} - actualStats := *cgroups.NewStats() - err := blkio.GetStats(helper.CgroupPath, &actualStats) - if err != nil { - t.Fatalf("Failed unexpectedly: %s", err) - } -} - -func TestBlkioStatsNoServiceBytesFile(t *testing.T) { - helper := NewCgroupTestUtil("blkio", t) - defer helper.cleanup() - helper.writeFileContents(map[string]string{ - "blkio.io_serviced_recursive": servicedRecursiveContents, - "blkio.io_queued_recursive": queuedRecursiveContents, - "blkio.sectors_recursive": sectorsRecursiveContents, - "blkio.io_service_time_recursive": serviceTimeRecursiveContents, - "blkio.io_wait_time_recursive": waitTimeRecursiveContents, - "blkio.io_merged_recursive": mergedRecursiveContents, - "blkio.time_recursive": timeRecursiveContents, - }) - - blkio := &BlkioGroup{} - actualStats := *cgroups.NewStats() - err := blkio.GetStats(helper.CgroupPath, &actualStats) - if err != nil { - t.Fatalf("Failed unexpectedly: %s", err) - } -} - -func TestBlkioStatsNoServicedFile(t *testing.T) { - helper := NewCgroupTestUtil("blkio", t) - defer helper.cleanup() - helper.writeFileContents(map[string]string{ - "blkio.io_service_bytes_recursive": serviceBytesRecursiveContents, - "blkio.io_queued_recursive": queuedRecursiveContents, - "blkio.sectors_recursive": sectorsRecursiveContents, - "blkio.io_service_time_recursive": serviceTimeRecursiveContents, - "blkio.io_wait_time_recursive": waitTimeRecursiveContents, - "blkio.io_merged_recursive": mergedRecursiveContents, - "blkio.time_recursive": timeRecursiveContents, - }) - - blkio := &BlkioGroup{} - actualStats := *cgroups.NewStats() - err := blkio.GetStats(helper.CgroupPath, &actualStats) - if err != nil { - t.Fatalf("Failed unexpectedly: %s", err) - } -} - -func TestBlkioStatsNoQueuedFile(t *testing.T) { - helper := NewCgroupTestUtil("blkio", t) - defer helper.cleanup() - helper.writeFileContents(map[string]string{ - "blkio.io_service_bytes_recursive": serviceBytesRecursiveContents, - "blkio.io_serviced_recursive": servicedRecursiveContents, - "blkio.sectors_recursive": sectorsRecursiveContents, - "blkio.io_service_time_recursive": serviceTimeRecursiveContents, - "blkio.io_wait_time_recursive": waitTimeRecursiveContents, - "blkio.io_merged_recursive": mergedRecursiveContents, - "blkio.time_recursive": timeRecursiveContents, - }) - - blkio := &BlkioGroup{} - actualStats := *cgroups.NewStats() - err := blkio.GetStats(helper.CgroupPath, &actualStats) - if err != nil { - t.Fatalf("Failed unexpectedly: %s", err) - } -} - -func TestBlkioStatsNoServiceTimeFile(t *testing.T) { +func TestBlkioStatsNoFilesCFQ(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } - helper := NewCgroupTestUtil("blkio", t) - defer helper.cleanup() - helper.writeFileContents(map[string]string{ - "blkio.io_service_bytes_recursive": serviceBytesRecursiveContents, - "blkio.io_serviced_recursive": servicedRecursiveContents, - "blkio.io_queued_recursive": queuedRecursiveContents, - "blkio.io_wait_time_recursive": waitTimeRecursiveContents, - "blkio.io_merged_recursive": mergedRecursiveContents, - "blkio.time_recursive": timeRecursiveContents, - "blkio.sectors_recursive": sectorsRecursiveContents, - }) - - blkio := &BlkioGroup{} - actualStats := *cgroups.NewStats() - err := blkio.GetStats(helper.CgroupPath, &actualStats) - if err != nil { - t.Fatalf("Failed unexpectedly: %s", err) + testCases := []blkioStatFailureTestCase{ + { + desc: "missing blkio.io_service_bytes_recursive file", + filename: "blkio.io_service_bytes_recursive", + }, + { + desc: "missing blkio.io_serviced_recursive file", + filename: "blkio.io_serviced_recursive", + }, + { + desc: "missing blkio.io_queued_recursive file", + filename: "blkio.io_queued_recursive", + }, + { + desc: "missing blkio.sectors_recursive file", + filename: "blkio.sectors_recursive", + }, + { + desc: "missing blkio.io_service_time_recursive file", + filename: "blkio.io_service_time_recursive", + }, + { + desc: "missing blkio.io_wait_time_recursive file", + filename: "blkio.io_wait_time_recursive", + }, + { + desc: "missing blkio.io_merged_recursive file", + filename: "blkio.io_merged_recursive", + }, + { + desc: "missing blkio.time_recursive file", + filename: "blkio.time_recursive", + }, } -} -func TestBlkioStatsNoWaitTimeFile(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - helper := NewCgroupTestUtil("blkio", t) - defer helper.cleanup() - helper.writeFileContents(map[string]string{ - "blkio.io_service_bytes_recursive": serviceBytesRecursiveContents, - "blkio.io_serviced_recursive": servicedRecursiveContents, - "blkio.io_queued_recursive": queuedRecursiveContents, - "blkio.io_service_time_recursive": serviceTimeRecursiveContents, - "blkio.io_merged_recursive": mergedRecursiveContents, - "blkio.time_recursive": timeRecursiveContents, - "blkio.sectors_recursive": sectorsRecursiveContents, - }) + for _, testCase := range testCases { + helper := NewCgroupTestUtil("cpuset", t) + defer helper.cleanup() - blkio := &BlkioGroup{} - actualStats := *cgroups.NewStats() - err := blkio.GetStats(helper.CgroupPath, &actualStats) - if err != nil { - t.Fatalf("Failed unexpectedly: %s", err) - } -} + tempBlkioTestFiles := map[string]string{} + for i, v := range blkioCFQStatsTestFiles { + tempBlkioTestFiles[i] = v + } + delete(tempBlkioTestFiles, testCase.filename) -func TestBlkioStatsNoMergedFile(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - helper := NewCgroupTestUtil("blkio", t) - defer helper.cleanup() - helper.writeFileContents(map[string]string{ - "blkio.io_service_bytes_recursive": serviceBytesRecursiveContents, - "blkio.io_serviced_recursive": servicedRecursiveContents, - "blkio.io_queued_recursive": queuedRecursiveContents, - "blkio.io_service_time_recursive": serviceTimeRecursiveContents, - "blkio.io_wait_time_recursive": waitTimeRecursiveContents, - "blkio.time_recursive": timeRecursiveContents, - "blkio.sectors_recursive": sectorsRecursiveContents, - }) + helper.writeFileContents(tempBlkioTestFiles) + cpuset := &CpusetGroup{} + actualStats := *cgroups.NewStats() + err := cpuset.GetStats(helper.CgroupPath, &actualStats) - blkio := &BlkioGroup{} - actualStats := *cgroups.NewStats() - err := blkio.GetStats(helper.CgroupPath, &actualStats) - if err != nil { - t.Fatalf("Failed unexpectedly: %s", err) - } -} - -func TestBlkioStatsNoTimeFile(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - helper := NewCgroupTestUtil("blkio", t) - defer helper.cleanup() - helper.writeFileContents(map[string]string{ - "blkio.io_service_bytes_recursive": serviceBytesRecursiveContents, - "blkio.io_serviced_recursive": servicedRecursiveContents, - "blkio.io_queued_recursive": queuedRecursiveContents, - "blkio.io_service_time_recursive": serviceTimeRecursiveContents, - "blkio.io_wait_time_recursive": waitTimeRecursiveContents, - "blkio.io_merged_recursive": mergedRecursiveContents, - "blkio.sectors_recursive": sectorsRecursiveContents, - }) - - blkio := &BlkioGroup{} - actualStats := *cgroups.NewStats() - err := blkio.GetStats(helper.CgroupPath, &actualStats) - if err != nil { - t.Fatalf("Failed unexpectedly: %s", err) + if err != nil { + t.Errorf(fmt.Sprintf("test case '%s' failed unexpectedly: %s", testCase.desc, err)) + } } } @@ -464,7 +571,58 @@ func TestBlkioStatsUnexpectedFieldType(t *testing.T) { } } -func TestNonCFQBlkioStats(t *testing.T) { +func TestThrottleRecursiveBlkioStats(t *testing.T) { + helper := NewCgroupTestUtil("blkio", t) + defer helper.cleanup() + helper.writeFileContents(map[string]string{ + "blkio.io_service_bytes_recursive": "", + "blkio.io_serviced_recursive": "", + "blkio.io_queued_recursive": "", + "blkio.sectors_recursive": "", + "blkio.io_service_time_recursive": "", + "blkio.io_wait_time_recursive": "", + "blkio.io_merged_recursive": "", + "blkio.time_recursive": "", + "blkio.throttle.io_service_bytes_recursive": throttleServiceBytesRecursive, + "blkio.throttle.io_serviced_recursive": throttleServicedRecursive, + }) + + blkio := &BlkioGroup{} + actualStats := *cgroups.NewStats() + err := blkio.GetStats(helper.CgroupPath, &actualStats) + if err != nil { + t.Fatal(err) + } + + // Verify expected stats. + expectedStats := cgroups.BlkioStats{} + + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 110305281, "Read") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 231, "Write") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 421, "Sync") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 110305281, "Async") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 110305281, "Total") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 110305281, "Read") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 231, "Write") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 421, "Sync") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 110305281, "Async") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 110305281, "Total") + + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 1641, "Read") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 231, "Write") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 421, "Sync") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 1641, "Async") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 1641, "Total") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 1641, "Read") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 231, "Write") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 421, "Sync") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 1641, "Async") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 1641, "Total") + + expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats) +} + +func TestThrottleBlkioStats(t *testing.T) { helper := NewCgroupTestUtil("blkio", t) defer helper.cleanup() helper.writeFileContents(map[string]string{ From 31b0151fc91d728a6356c4a79e3af75d041fea6a Mon Sep 17 00:00:00 2001 From: Piotr Wagner Date: Mon, 24 Aug 2020 21:04:40 +0200 Subject: [PATCH 2/2] move blkio stat gathering to loop Signed-off-by: Piotr Wagner --- libcontainer/cgroups/fs/blkio.go | 265 ++++++++++++-------------- libcontainer/cgroups/fs/blkio_test.go | 56 ++++++ 2 files changed, 173 insertions(+), 148 deletions(-) diff --git a/libcontainer/cgroups/fs/blkio.go b/libcontainer/cgroups/fs/blkio.go index 593a4b945..dca2a6220 100644 --- a/libcontainer/cgroups/fs/blkio.go +++ b/libcontainer/cgroups/fs/blkio.go @@ -157,165 +157,134 @@ func getBlkioStat(dir, file string) ([]cgroups.BlkioStatEntry, error) { } func (s *BlkioGroup) GetStats(path string, stats *cgroups.Stats) error { - // Try to read BFQ stats with debug files enabled first - if moveToNextMethod, err := getBFQDebugStats(path, stats); !moveToNextMethod { - return err + type blkioStatInfo struct { + filename string + blkioStatEntriesPtr *[]cgroups.BlkioStatEntry } - // Try to read BFQ stats with debug files disabled - if moveToNextMethod, err := getBFQStats(path, stats); !moveToNextMethod { - return err + var bfqDebugStats = []blkioStatInfo{ + { + filename: "blkio.bfq.sectors_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.SectorsRecursive, + }, + { + filename: "blkio.bfq.io_service_time_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoServiceTimeRecursive, + }, + { + filename: "blkio.bfq.io_wait_time_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoWaitTimeRecursive, + }, + { + filename: "blkio.bfq.io_merged_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoMergedRecursive, + }, + { + filename: "blkio.bfq.io_queued_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoQueuedRecursive, + }, + { + filename: "blkio.bfq.time_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoTimeRecursive, + }, + { + filename: "blkio.bfq.io_serviced_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive, + }, + { + filename: "blkio.bfq.io_service_bytes_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive, + }, } - // Try to read CFQ stats available on all CFQ enabled kernels - if moveToNextMethod, err := getCFQStats(path, stats); !moveToNextMethod { - return err + var bfqStats = []blkioStatInfo{ + { + filename: "blkio.bfq.io_serviced_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive, + }, + { + filename: "blkio.bfq.io_service_bytes_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive, + }, } - // Try to read recursive version of generic stats - if moveToNextMethod, err := getRecursiveThrottleStats(path, stats); !moveToNextMethod { - return err + var cfqStats = []blkioStatInfo{ + { + filename: "blkio.sectors_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.SectorsRecursive, + }, + { + filename: "blkio.io_service_time_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoServiceTimeRecursive, + }, + { + filename: "blkio.io_wait_time_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoWaitTimeRecursive, + }, + { + filename: "blkio.io_merged_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoMergedRecursive, + }, + { + filename: "blkio.io_queued_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoQueuedRecursive, + }, + { + filename: "blkio.time_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoTimeRecursive, + }, + { + filename: "blkio.io_serviced_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive, + }, + { + filename: "blkio.io_service_bytes_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive, + }, } - return getBaseStats(path, stats) // Use generic stats as fallback -} - -func getBFQStats(path string, stats *cgroups.Stats) (bool, error) { - var blkioStats []cgroups.BlkioStatEntry - var err error - var moveToNextMethod = false - - if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_serviced_recursive"); err != nil || blkioStats == nil { - moveToNextMethod = true - return moveToNextMethod, err + var throttleRecursiveStats = []blkioStatInfo{ + { + filename: "blkio.throttle.io_serviced_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive, + }, + { + filename: "blkio.throttle.io_service_bytes_recursive", + blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive, + }, } - stats.BlkioStats.IoServicedRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_service_bytes_recursive"); err != nil { - return moveToNextMethod, err + var baseStats = []blkioStatInfo{ + { + filename: "blkio.throttle.io_serviced", + blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive, + }, + { + filename: "blkio.throttle.io_service_bytes", + blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive, + }, } - stats.BlkioStats.IoServiceBytesRecursive = blkioStats - - return moveToNextMethod, nil -} - -func getBFQDebugStats(path string, stats *cgroups.Stats) (bool, error) { - var blkioStats []cgroups.BlkioStatEntry - var err error - var moveToNextMethod = false - - if blkioStats, err = getBlkioStat(path, "blkio.bfq.sectors_recursive"); err != nil || blkioStats == nil { - moveToNextMethod = true - return moveToNextMethod, err + var orderedStats = [][]blkioStatInfo{ + bfqDebugStats, + bfqStats, + cfqStats, + throttleRecursiveStats, + baseStats, } - stats.BlkioStats.SectorsRecursive = blkioStats - if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_service_time_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoServiceTimeRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_wait_time_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoWaitTimeRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_merged_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoMergedRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.bfq.io_queued_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoQueuedRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.bfq.time_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoTimeRecursive = blkioStats - - _, err = getBFQStats(path, stats) - return moveToNextMethod, err -} - -func getCFQStats(path string, stats *cgroups.Stats) (bool, error) { - var blkioStats []cgroups.BlkioStatEntry - var err error - var moveToNextMethod = false - - if blkioStats, err = getBlkioStat(path, "blkio.sectors_recursive"); err != nil || blkioStats == nil { - moveToNextMethod = true - return moveToNextMethod, err - } - stats.BlkioStats.SectorsRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.io_service_bytes_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoServiceBytesRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.io_serviced_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoServicedRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.io_queued_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoQueuedRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.io_service_time_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoServiceTimeRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.io_wait_time_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoWaitTimeRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.io_merged_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoMergedRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.time_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoTimeRecursive = blkioStats - - return moveToNextMethod, nil -} - -func getRecursiveThrottleStats(path string, stats *cgroups.Stats) (bool, error) { - var blkioStats []cgroups.BlkioStatEntry - var err error - var moveToNextMethod = false - - if blkioStats, err = getBlkioStat(path, "blkio.throttle.io_service_bytes_recursive"); err != nil || blkioStats == nil { - moveToNextMethod = true - return moveToNextMethod, err - } - stats.BlkioStats.IoServiceBytesRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.throttle.io_serviced_recursive"); err != nil { - return moveToNextMethod, err - } - stats.BlkioStats.IoServicedRecursive = blkioStats - - return moveToNextMethod, nil -} - -func getBaseStats(path string, stats *cgroups.Stats) error { var blkioStats []cgroups.BlkioStatEntry var err error - if blkioStats, err = getBlkioStat(path, "blkio.throttle.io_service_bytes"); err != nil { - return err + for _, statGroup := range orderedStats { + for i, statInfo := range statGroup { + if blkioStats, err = getBlkioStat(path, statInfo.filename); err != nil || blkioStats == nil { + // if error occurs on first file, move to next group + if i == 0 { + break + } + return err + } + *statInfo.blkioStatEntriesPtr = blkioStats + //finish if all stats are gathered + if i == len(statGroup)-1 { + return nil + } + } } - stats.BlkioStats.IoServiceBytesRecursive = blkioStats - - if blkioStats, err = getBlkioStat(path, "blkio.throttle.io_serviced"); err != nil { - return err - } - stats.BlkioStats.IoServicedRecursive = blkioStats - return nil } diff --git a/libcontainer/cgroups/fs/blkio_test.go b/libcontainer/cgroups/fs/blkio_test.go index d26b9d23c..235648052 100644 --- a/libcontainer/cgroups/fs/blkio_test.go +++ b/libcontainer/cgroups/fs/blkio_test.go @@ -322,6 +322,62 @@ func TestBlkioBFQDebugStats(t *testing.T) { expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats) } +func TestBlkioMultipleStatsFiles(t *testing.T) { + helper := NewCgroupTestUtil("blkio", t) + defer helper.cleanup() + helper.writeFileContents(blkioBFQDebugStatsTestFiles) + helper.writeFileContents(blkioCFQStatsTestFiles) + blkio := &BlkioGroup{} + actualStats := *cgroups.NewStats() + err := blkio.GetStats(helper.CgroupPath, &actualStats) + if err != nil { + t.Fatal(err) + } + + expectedStats := cgroups.BlkioStats{} + appendBlkioStatEntry(&expectedStats.SectorsRecursive, 8, 0, 2048, "") + + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1100, "Read") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1200, "Write") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1300, "Sync") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Async") + appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Total") + + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 11, "Read") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 41, "Write") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 21, "Sync") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 31, "Async") + appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 51, "Total") + + appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 2, "Read") + appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 3, "Write") + appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 4, "Sync") + appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 5, "Async") + appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 6, "Total") + + appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Read") + appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Write") + appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Sync") + appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173, "Async") + appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 174, "Total") + + appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Read") + appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Write") + appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Sync") + appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Async") + appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Total") + + appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 51, "Read") + appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 101, "Write") + appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Sync") + appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Async") + appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 151, "Total") + + appendBlkioStatEntry(&expectedStats.IoTimeRecursive, 8, 0, 16, "") + + expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats) +} + func TestBlkioBFQStats(t *testing.T) { helper := NewCgroupTestUtil("blkio", t) defer helper.cleanup()