Merge pull request #2407 from piowag/2365-blkio-getstat-update

Update blkio GetStats
2026-07-11 06:03:57 +08:00 · 2020-12-03 15:46:20 -08:00
parent bd013b6e4d 31b0151fc9
commit 8518317c39
2 changed files with 520 additions and 245 deletions
@@ -157,73 +157,134 @@ func getBlkioStat(dir, file string) ([]cgroups.BlkioStatEntry, error) {
 }

 func (s *BlkioGroup) GetStats(path string, stats *cgroups.Stats) error {
-	// Try to read CFQ stats available on all CFQ enabled kernels first
-	if blkioStats, err := getBlkioStat(path, "blkio.io_serviced_recursive"); err == nil && blkioStats != nil {
-		return getCFQStats(path, stats)
+	type blkioStatInfo struct {
+		filename            string
+		blkioStatEntriesPtr *[]cgroups.BlkioStatEntry
+	}
+	var bfqDebugStats = []blkioStatInfo{
+		{
+			filename:            "blkio.bfq.sectors_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.SectorsRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_service_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceTimeRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_wait_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoWaitTimeRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_merged_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoMergedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_queued_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoQueuedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoTimeRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
+	}
+	var bfqStats = []blkioStatInfo{
+		{
+			filename:            "blkio.bfq.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
+	}
+	var cfqStats = []blkioStatInfo{
+		{
+			filename:            "blkio.sectors_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.SectorsRecursive,
+		},
+		{
+			filename:            "blkio.io_service_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceTimeRecursive,
+		},
+		{
+			filename:            "blkio.io_wait_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoWaitTimeRecursive,
+		},
+		{
+			filename:            "blkio.io_merged_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoMergedRecursive,
+		},
+		{
+			filename:            "blkio.io_queued_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoQueuedRecursive,
+		},
+		{
+			filename:            "blkio.time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoTimeRecursive,
+		},
+		{
+			filename:            "blkio.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
+	}
+	var throttleRecursiveStats = []blkioStatInfo{
+		{
+			filename:            "blkio.throttle.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.throttle.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
+	}
+	var baseStats = []blkioStatInfo{
+		{
+			filename:            "blkio.throttle.io_serviced",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.throttle.io_service_bytes",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
+	}
+	var orderedStats = [][]blkioStatInfo{
+		bfqDebugStats,
+		bfqStats,
+		cfqStats,
+		throttleRecursiveStats,
+		baseStats,
 	}
-	return getStats(path, stats) // Use generic stats as fallback
-}

-func getCFQStats(path string, stats *cgroups.Stats) error {
 	var blkioStats []cgroups.BlkioStatEntry
 	var err error

-	if blkioStats, err = getBlkioStat(path, "blkio.sectors_recursive"); err != nil {
-		return err
+	for _, statGroup := range orderedStats {
+		for i, statInfo := range statGroup {
+			if blkioStats, err = getBlkioStat(path, statInfo.filename); err != nil || blkioStats == nil {
+				// if error occurs on first file, move to next group
+				if i == 0 {
+					break
+				}
+				return err
+			}
+			*statInfo.blkioStatEntriesPtr = blkioStats
+			//finish if all stats are gathered
+			if i == len(statGroup)-1 {
+				return nil
+			}
+		}
 	}
-	stats.BlkioStats.SectorsRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_service_bytes_recursive"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServiceBytesRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_serviced_recursive"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServicedRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_queued_recursive"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoQueuedRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_service_time_recursive"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServiceTimeRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_wait_time_recursive"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoWaitTimeRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_merged_recursive"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoMergedRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.time_recursive"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoTimeRecursive = blkioStats
-
-	return nil
-}
-
-func getStats(path string, stats *cgroups.Stats) error {
-	var blkioStats []cgroups.BlkioStatEntry
-	var err error
-
-	if blkioStats, err = getBlkioStat(path, "blkio.throttle.io_service_bytes"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServiceBytesRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.throttle.io_serviced"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServicedRecursive = blkioStats
-
 	return nil
 }
@@ -3,6 +3,7 @@
 package fs

 import (
+	"fmt"
 	"strconv"
 	"testing"

@@ -13,43 +14,81 @@ import (

 const (
 	sectorsRecursiveContents      = `8:0 1024`
+	sectorsRecursiveContentsBFQ   = `8:0 2048`
 	serviceBytesRecursiveContents = `8:0 Read 100
 8:0 Write 200
 8:0 Sync 300
 8:0 Async 500
 8:0 Total 500
 Total 500`
+
+	serviceBytesRecursiveContentsBFQ = `8:0 Read 1100
+8:0 Write 1200
+8:0 Sync 1300
+8:0 Async 1500
+8:0 Total 1500
+Total 1500`
 	servicedRecursiveContents = `8:0 Read 10
 8:0 Write 40
 8:0 Sync 20
 8:0 Async 30
 8:0 Total 50
 Total 50`
+	servicedRecursiveContentsBFQ = `8:0 Read 11
+8:0 Write 41
+8:0 Sync 21
+8:0 Async 31
+8:0 Total 51
+Total 51`
 	queuedRecursiveContents = `8:0 Read 1
 8:0 Write 4
 8:0 Sync 2
 8:0 Async 3
 8:0 Total 5
 Total 5`
+	queuedRecursiveContentsBFQ = `8:0 Read 2
+8:0 Write 3
+8:0 Sync 4
+8:0 Async 5
+8:0 Total 6
+Total 6`
 	serviceTimeRecursiveContents = `8:0 Read 173959
 8:0 Write 0
 8:0 Sync 0
 8:0 Async 173959
 8:0 Total 17395
 Total 17395`
+	serviceTimeRecursiveContentsBFQ = `8:0 Read 173959
+8:0 Write 0
+8:0 Sync 0
+8:0 Async 173
+8:0 Total 174
+Total 174`
 	waitTimeRecursiveContents = `8:0 Read 15571
 8:0 Write 0
 8:0 Sync 0
 8:0 Async 15571
 8:0 Total 15571`
+	waitTimeRecursiveContentsBFQ = `8:0 Read 1557
+8:0 Write 0
+8:0 Sync 0
+8:0 Async 1557
+8:0 Total 1557`
 	mergedRecursiveContents = `8:0 Read 5
 8:0 Write 10
 8:0 Sync 0
 8:0 Async 0
 8:0 Total 15
 Total 15`
-	timeRecursiveContents = `8:0 8`
-	throttleServiceBytes  = `8:0 Read 11030528
+	mergedRecursiveContentsBFQ = `8:0 Read 51
+8:0 Write 101
+8:0 Sync 0
+8:0 Async 0
+8:0 Total 151
+Total 151`
+	timeRecursiveContents    = `8:0 8`
+	timeRecursiveContentsBFQ = `8:0 16`
+	throttleServiceBytes     = `8:0 Read 11030528
 8:0 Write 23
 8:0 Sync 42
 8:0 Async 11030528
@@ -60,6 +99,17 @@ Total 15`
 252:0 Async 11030528
 252:0 Total 11030528
 Total 22061056`
+	throttleServiceBytesRecursive = `8:0 Read 110305281
+8:0 Write 231
+8:0 Sync 421
+8:0 Async 110305281
+8:0 Total 110305281
+252:0 Read 110305281
+252:0 Write 231
+252:0 Sync 421
+252:0 Async 110305281
+252:0 Total 110305281
+Total 220610561`
 	throttleServiced = `8:0 Read 164
 8:0 Write 23
 8:0 Sync 42
@@ -71,8 +121,51 @@ Total 22061056`
 252:0 Async 164
 252:0 Total 164
 Total 328`
+	throttleServicedRecursive = `8:0 Read 1641
+8:0 Write 231
+8:0 Sync 421
+8:0 Async 1641
+8:0 Total 1641
+252:0 Read 1641
+252:0 Write 231
+252:0 Sync 421
+252:0 Async 1641
+252:0 Total 1641
+Total 3281`
 )

+var blkioBFQDebugStatsTestFiles = map[string]string{
+	"blkio.bfq.io_service_bytes_recursive": serviceBytesRecursiveContentsBFQ,
+	"blkio.bfq.io_serviced_recursive":      servicedRecursiveContentsBFQ,
+	"blkio.bfq.io_queued_recursive":        queuedRecursiveContentsBFQ,
+	"blkio.bfq.io_service_time_recursive":  serviceTimeRecursiveContentsBFQ,
+	"blkio.bfq.io_wait_time_recursive":     waitTimeRecursiveContentsBFQ,
+	"blkio.bfq.io_merged_recursive":        mergedRecursiveContentsBFQ,
+	"blkio.bfq.time_recursive":             timeRecursiveContentsBFQ,
+	"blkio.bfq.sectors_recursive":          sectorsRecursiveContentsBFQ,
+}
+
+var blkioBFQStatsTestFiles = map[string]string{
+	"blkio.bfq.io_service_bytes_recursive": serviceBytesRecursiveContentsBFQ,
+	"blkio.bfq.io_serviced_recursive":      servicedRecursiveContentsBFQ,
+}
+
+var blkioCFQStatsTestFiles = map[string]string{
+	"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+	"blkio.io_serviced_recursive":      servicedRecursiveContents,
+	"blkio.io_queued_recursive":        queuedRecursiveContents,
+	"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+	"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+	"blkio.io_merged_recursive":        mergedRecursiveContents,
+	"blkio.time_recursive":             timeRecursiveContents,
+	"blkio.sectors_recursive":          sectorsRecursiveContents,
+}
+
+type blkioStatFailureTestCase struct {
+	desc     string
+	filename string
+}
+
 func appendBlkioStatEntry(blkioStatEntries *[]cgroups.BlkioStatEntry, major, minor, value uint64, op string) {
 	*blkioStatEntries = append(*blkioStatEntries, cgroups.BlkioStatEntry{Major: major, Minor: minor, Value: value, Op: op})
 }
@@ -174,19 +267,209 @@ func TestBlkioSetMultipleWeightDevice(t *testing.T) {
 	}
 }

-func TestBlkioStats(t *testing.T) {
+func TestBlkioBFQDebugStats(t *testing.T) {
 	helper := NewCgroupTestUtil("blkio", t)
 	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-	})
+	helper.writeFileContents(blkioBFQDebugStatsTestFiles)
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedStats := cgroups.BlkioStats{}
+	appendBlkioStatEntry(&expectedStats.SectorsRecursive, 8, 0, 2048, "")
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1100, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1200, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1300, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 11, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 41, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 21, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 31, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 51, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 2, "Read")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 3, "Write")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 4, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 5, "Async")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 6, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 174, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Read")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Write")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Async")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 51, "Read")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 101, "Write")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Async")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 151, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoTimeRecursive, 8, 0, 16, "")
+
+	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
+}
+
+func TestBlkioMultipleStatsFiles(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(blkioBFQDebugStatsTestFiles)
+	helper.writeFileContents(blkioCFQStatsTestFiles)
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedStats := cgroups.BlkioStats{}
+	appendBlkioStatEntry(&expectedStats.SectorsRecursive, 8, 0, 2048, "")
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1100, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1200, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1300, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 11, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 41, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 21, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 31, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 51, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 2, "Read")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 3, "Write")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 4, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 5, "Async")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 6, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 174, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Read")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Write")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Async")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 51, "Read")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 101, "Write")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Async")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 151, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoTimeRecursive, 8, 0, 16, "")
+
+	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
+}
+
+func TestBlkioBFQStats(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(blkioBFQStatsTestFiles)
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedStats := cgroups.BlkioStats{}
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1100, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1200, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1300, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 11, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 41, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 21, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 31, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 51, "Total")
+
+	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
+}
+
+func TestBlkioStatsNoFilesBFQDebug(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+	testCases := []blkioStatFailureTestCase{
+		{
+			desc:     "missing blkio.bfq.io_service_bytes_recursive file",
+			filename: "blkio.bfq.io_service_bytes_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.io_serviced_recursive file",
+			filename: "blkio.bfq.io_serviced_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.io_queued_recursive file",
+			filename: "blkio.bfq.io_queued_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.sectors_recursive file",
+			filename: "blkio.bfq.sectors_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.io_service_time_recursive file",
+			filename: "blkio.bfq.io_service_time_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.io_wait_time_recursive file",
+			filename: "blkio.bfq.io_wait_time_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.io_merged_recursive file",
+			filename: "blkio.bfq.io_merged_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.time_recursive file",
+			filename: "blkio.bfq.time_recursive",
+		},
+	}
+
+	for _, testCase := range testCases {
+		helper := NewCgroupTestUtil("cpuset", t)
+		defer helper.cleanup()
+
+		tempBlkioTestFiles := map[string]string{}
+		for i, v := range blkioBFQDebugStatsTestFiles {
+			tempBlkioTestFiles[i] = v
+		}
+		delete(tempBlkioTestFiles, testCase.filename)
+
+		helper.writeFileContents(tempBlkioTestFiles)
+		cpuset := &CpusetGroup{}
+		actualStats := *cgroups.NewStats()
+		err := cpuset.GetStats(helper.CgroupPath, &actualStats)
+
+		if err != nil {
+			t.Errorf(fmt.Sprintf("test case '%s' failed unexpectedly: %s", testCase.desc, err))
+		}
+	}
+}
+
+func TestBlkioCFQStats(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(blkioCFQStatsTestFiles)

 	blkio := &BlkioGroup{}
 	actualStats := *cgroups.NewStats()
@@ -240,183 +523,63 @@ func TestBlkioStats(t *testing.T) {
 	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
 }

-func TestBlkioStatsNoSectorsFile(t *testing.T) {
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-	})
-
-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
-	}
-}
-
-func TestBlkioStatsNoServiceBytesFile(t *testing.T) {
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_serviced_recursive":     servicedRecursiveContents,
-		"blkio.io_queued_recursive":       queuedRecursiveContents,
-		"blkio.sectors_recursive":         sectorsRecursiveContents,
-		"blkio.io_service_time_recursive": serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":    waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":       mergedRecursiveContents,
-		"blkio.time_recursive":            timeRecursiveContents,
-	})
-
-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
-	}
-}
-
-func TestBlkioStatsNoServicedFile(t *testing.T) {
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-	})
-
-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
-	}
-}
-
-func TestBlkioStatsNoQueuedFile(t *testing.T) {
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-	})
-
-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
-	}
-}
-
-func TestBlkioStatsNoServiceTimeFile(t *testing.T) {
+func TestBlkioStatsNoFilesCFQ(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode.")
 	}
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-	})
-
-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
+	testCases := []blkioStatFailureTestCase{
+		{
+			desc:     "missing blkio.io_service_bytes_recursive file",
+			filename: "blkio.io_service_bytes_recursive",
+		},
+		{
+			desc:     "missing blkio.io_serviced_recursive file",
+			filename: "blkio.io_serviced_recursive",
+		},
+		{
+			desc:     "missing blkio.io_queued_recursive file",
+			filename: "blkio.io_queued_recursive",
+		},
+		{
+			desc:     "missing blkio.sectors_recursive file",
+			filename: "blkio.sectors_recursive",
+		},
+		{
+			desc:     "missing blkio.io_service_time_recursive file",
+			filename: "blkio.io_service_time_recursive",
+		},
+		{
+			desc:     "missing blkio.io_wait_time_recursive file",
+			filename: "blkio.io_wait_time_recursive",
+		},
+		{
+			desc:     "missing blkio.io_merged_recursive file",
+			filename: "blkio.io_merged_recursive",
+		},
+		{
+			desc:     "missing blkio.time_recursive file",
+			filename: "blkio.time_recursive",
+		},
 	}
-}

-func TestBlkioStatsNoWaitTimeFile(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode.")
-	}
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-	})
+	for _, testCase := range testCases {
+		helper := NewCgroupTestUtil("cpuset", t)
+		defer helper.cleanup()

-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
-	}
-}
+		tempBlkioTestFiles := map[string]string{}
+		for i, v := range blkioCFQStatsTestFiles {
+			tempBlkioTestFiles[i] = v
+		}
+		delete(tempBlkioTestFiles, testCase.filename)

-func TestBlkioStatsNoMergedFile(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode.")
-	}
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-	})
+		helper.writeFileContents(tempBlkioTestFiles)
+		cpuset := &CpusetGroup{}
+		actualStats := *cgroups.NewStats()
+		err := cpuset.GetStats(helper.CgroupPath, &actualStats)

-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
-	}
-}
-
-func TestBlkioStatsNoTimeFile(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode.")
-	}
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-	})
-
-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
+		if err != nil {
+			t.Errorf(fmt.Sprintf("test case '%s' failed unexpectedly: %s", testCase.desc, err))
+		}
 	}
 }

@@ -464,7 +627,58 @@ func TestBlkioStatsUnexpectedFieldType(t *testing.T) {
 	}
 }

-func TestNonCFQBlkioStats(t *testing.T) {
+func TestThrottleRecursiveBlkioStats(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive":          "",
+		"blkio.io_serviced_recursive":               "",
+		"blkio.io_queued_recursive":                 "",
+		"blkio.sectors_recursive":                   "",
+		"blkio.io_service_time_recursive":           "",
+		"blkio.io_wait_time_recursive":              "",
+		"blkio.io_merged_recursive":                 "",
+		"blkio.time_recursive":                      "",
+		"blkio.throttle.io_service_bytes_recursive": throttleServiceBytesRecursive,
+		"blkio.throttle.io_serviced_recursive":      throttleServicedRecursive,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify expected stats.
+	expectedStats := cgroups.BlkioStats{}
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 110305281, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 231, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 421, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 110305281, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 110305281, "Total")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 110305281, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 231, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 421, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 110305281, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 110305281, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 1641, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 231, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 421, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 1641, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 1641, "Total")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 1641, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 231, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 421, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 1641, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 1641, "Total")
+
+	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
+}
+
+func TestThrottleBlkioStats(t *testing.T) {
 	helper := NewCgroupTestUtil("blkio", t)
 	defer helper.cleanup()
 	helper.writeFileContents(map[string]string{