chore: update turbostat

cmd/telemetry/telemetry_tables.go

@@ -387,15 +387,15 @@ func powerTelemetryTableValues(outputs map[string]script.ScriptOutput) []table.F
 	// dynamically build fields from the header row of the first package
 	// header format: ["timestamp", "PkgWatt", "RAMWatt", ...]
 	// fields will be: "Time", then for each package, one field per matched watt metric
-	// e.g., "Package 0 PkgWatt", "Package 0 RAMWatt", "Package 1 PkgWatt", ...
+	// e.g., "Pkg 0 PkgWatt", "Pkg 0 RAMWatt", "Pkg 1 PkgWatt", ...
 	header := packageRows[0][0]
 	fields := []table.Field{{Name: "Time"}}
 	for i, pkgRows := range packageRows {
 		if len(pkgRows) == 0 {
 			continue
 		}
 		for _, fieldName := range header[1:] {
-			fields = append(fields, table.Field{Name: fmt.Sprintf("Package %d %s", i, fieldName)})
+			fields = append(fields, table.Field{Name: fmt.Sprintf("Pkg %d %s", i, fieldName)})
 		}
 	}
 	numMetrics := len(header) - 1 // number of matched watt fields per package
@@ -458,28 +458,38 @@ func frequencyTelemetryTableValues(outputs map[string]script.ScriptOutput) []tab
 		slog.Warn(err.Error())
 		return []table.Field{}
 	}
-	packageRows, err := extract.TurbostatPackageRows(outputs[script.TurbostatTelemetryScriptName].Stdout, []string{"UncMHz"})
+	// UncMHz, UMHz0.0, UMHz1.0, etc. are the uncore frequency fields in turbostat output, but they aren't always present, so we look for any field that matches the regex
+	uncoreRegex := regexp.MustCompile(`^(UncMHz|UMHz\d+\.\d+)$`)
+	packageRows, err := extract.TurbostatPackageRowsByRegexMatch(outputs[script.TurbostatTelemetryScriptName].Stdout, []*regexp.Regexp{uncoreRegex})
 	if err != nil {
 		// not an error, just means no package rows (uncore frequency)
 		slog.Warn(err.Error())
 	}
-	// add the package rows to the fields
-	for i := range packageRows {
-		fields = append(fields, table.Field{Name: fmt.Sprintf("Uncore Package %d", i)})
-	}
 	// for each platform row
 	for i := range platformRows {
 		// append the timestamp to the fields
 		fields[0].Values = append(fields[0].Values, platformRows[i][0]) // Timestamp
 		// append the core frequency values to the fields
 		fields[1].Values = append(fields[1].Values, platformRows[i][1]) // Core frequency
 	}
-	// for each package
+	// dynamically build fields for uncore frequencies based on the package rows we found that match the uncore frequency regex
 	for i := range packageRows {
-		// traverse the rows
-		for _, row := range packageRows[i] {
-			// append the package frequency to the fields
-			fields[i+2].Values = append(fields[i+2].Values, row[1]) // Package frequency
+		// the first package row contains the field names, so use that to build the fields
+		for _, fieldName := range packageRows[i][0][1:] { // skip timestamp field
+			fields = append(fields, table.Field{Name: fmt.Sprintf("Pkg %d %s", i, fieldName)})
+		}
+	}
+	if len(packageRows) > 0 {
+		numUncoreMetrics := (len(fields) - 2) / len(packageRows) // number of uncore frequency fields per package, after time and core frequency fields
+		// append the uncore frequency values to the fields
+		for i := range packageRows {
+			for _, row := range packageRows[i][1:] { // skip header row
+				// append the package frequency to the fields
+				// uncore frequency fields start after time and core frequency fields
+				for j := range row[1:] { // skip timestamp field
+					fields[2+i*numUncoreMetrics+j].Values = append(fields[2+i*numUncoreMetrics+j].Values, row[j+1])
+				}
+			}
 		}
 	}
 	if len(fields[0].Values) == 0 {

internal/extract/turbostat.go

@@ -62,12 +62,12 @@ func parseTurbostatOutput(output string) ([]map[string]string, error) {
 		if len(headers) == 0 {
 			continue
 		}
-		if len(fields) != len(headers) {
-			continue
+		if len(fields) > len(headers) {
+			return nil, fmt.Errorf("more turbostat row values than headers: %d values, %d headers", len(fields), len(headers))
 		}
-		row := make(map[string]string, len(headers))
-		for i, h := range headers {
-			row[h] = fields[i]
+		row := make(map[string]string, len(fields))
+		for i := range fields {
+			row[headers[i]] = fields[i] // this assumes any missing fields are at the end and will be empty string, which is consistent with turbostat output
 		}
 		if timeParsed && interval > 0 {
 			row["timestamp"] = timestamp.Format("15:04:05")
@@ -219,9 +219,16 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs
 	if len(rows) == 0 {
 		return nil, fmt.Errorf("no rows found in turbostat output")
 	}
+	// filter all rows down to only package rows
+	rows, err = extractPackageRows(rows)
+	if err != nil {
+		return nil, fmt.Errorf("unable to extract package rows: %w", err)
+	}
+	if len(rows) == 0 {
+		return nil, fmt.Errorf("no package rows found in turbostat output")
+	}
 	// Build our list of matched field names from the first package row
 	var matchedFields []string
-	foundPackageRow := false
 	for _, row := range rows {
 		if _, ok := row["Package"]; !ok {
 			if row["CPU"] == "0" {
@@ -230,10 +237,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs
 				continue
 			}
 		}
-		if !isPackageRow(row) {
-			continue
-		}
-		foundPackageRow = true
 		for field := range row {
 			for _, re := range fieldRegexs {
 				if re.MatchString(field) {
@@ -246,9 +249,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs
 		}
 		break // only need the first package row to discover fields
 	}
-	if !foundPackageRow {
-		return nil, fmt.Errorf("no package rows found in turbostat output")
-	}
 	if len(matchedFields) == 0 {
 		return nil, fmt.Errorf("no fields matched the provided regexes in turbostat output")
 	}
@@ -268,9 +268,6 @@ func TurbostatPackageRowsByRegexMatch(turboStatScriptOutput string, fieldRegexs
 				continue
 			}
 		}
-		if !isPackageRow(row) {
-			continue
-		}
 		rowValues := make([]string, len(matchedFields)+1)
 		rowValues[0] = row["timestamp"]
 		for i, field := range matchedFields {
@@ -322,6 +319,14 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([]
 	if len(rows) == 0 {
 		return nil, fmt.Errorf("no package rows found in turbostat output")
 	}
+	// filter all rows down to only package rows
+	rows, err = extractPackageRows(rows)
+	if err != nil {
+		return nil, fmt.Errorf("unable to extract package rows: %w", err)
+	}
+	if len(rows) == 0 {
+		return nil, fmt.Errorf("no package rows found in turbostat output")
+	}
 	var packageRows [][][]string
 	for _, row := range rows {
 		if _, ok := row["Package"]; !ok {
@@ -331,9 +336,6 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([]
 				continue
 			}
 		}
-		if !isPackageRow(row) {
-			continue
-		}
 		rowValues := make([]string, len(fieldNames)+1)
 		rowValues[0] = row["timestamp"]
 		for i, fieldName := range fieldNames {
@@ -359,11 +361,18 @@ func TurbostatPackageRows(turboStatScriptOutput string, fieldNames []string) ([]
 	return packageRows, nil
 }
 
-func isPackageRow(row map[string]string) bool {
-	if val, ok := row["Package"]; ok && val != "-" {
-		return true
+func extractPackageRows(rows []map[string]string) ([]map[string]string, error) {
+	var packageRows []map[string]string
+	for i, row := range rows {
+		if val, ok := row["Package"]; ok && val != "-" && row["Core"] == "0" {
+			if i > 0 && rows[i-1]["Package"] == val && rows[i-1]["Core"] == "0" {
+				// This is the hyperthread associated with the package row, skip it
+				continue
+			}
+			packageRows = append(packageRows, row)
+		}
 	}
-	return false
+	return packageRows, nil
 }
 
 // MaxTotalPackagePowerFromOutput calculates the maximum total package power from the turbostat output.

internal/extract/turbostat_test.go

@@ -442,16 +442,6 @@ func TestTurbostatPackageRows(t *testing.T) {
 		want            [][][]string
 		wantErr         bool
 	}{
-		{
-			name:            "package rows with UncoreMHz, PKGTmp, PkgWatt",
-			turbostatOutput: turbostatOutput,
-			fieldNames:      []string{"UncMHz", "PkgTmp", "PkgWatt"},
-			want: [][][]string{
-				{{"15:04:05", "2350", "57", "223.53"}, {"15:04:07", "2400", "59", "229.53"}, {"15:04:09", "2400", "57", "223.53"}},
-				{{"15:04:05", "2300", "53", "208.40"}, {"15:04:07", "2400", "55", "218.40"}, {"15:04:09", "2400", "53", "208.40"}},
-			},
-			wantErr: false,
-		},
 		{
 			name: "Typical output, two packages, one field",
 			turbostatOutput: `

tools/Makefile

@@ -25,7 +25,7 @@ default: tools-x86_64
 .PHONY: sysstat sysstat-aarch64 sysstat-repo
 .PHONY: tsc turbostat
 
-tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc turbostat
+tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc
 	mkdir -p bin/x86_64
 	cp -R async-profiler bin/x86_64/
 	cp avx-turbo/avx-turbo bin/x86_64/
@@ -50,7 +50,6 @@ tools-x86_64: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw
 	cp sysstat/sar bin/x86_64/
 	cp sysstat/sadc bin/x86_64/
 	cp tsc/tsc bin/x86_64/
-	cp linux_turbostat/tools/power/x86/turbostat/turbostat bin/x86_64/
 	cd bin/x86_64 && strip --strip-unneeded * || true
 
 tools-aarch64: async-profiler-aarch64 dmidecode-aarch64 ethtool-aarch64 fio-aarch64 ipmitool-aarch64 lshw-aarch64 lspci-aarch64 spectre-meltdown-checker sshpass-aarch64 stackcollapse-perf-aarch64 stress-ng-aarch64 sysstat-aarch64
@@ -190,9 +189,12 @@ else
 endif
 
 LIBAIO_VERSION := libaio-0.3.113
+PRIMARY_REPO="https://github.com/littledan/linux-libaio.git"
+FALLBACK_REPO="https://github.com/yugabyte/libaio.git"
 libaio-aarch64:
 ifeq ("$(wildcard libaio-aarch64)","")
-	git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) https://pagure.io/libaio libaio-aarch64
+	git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) $(PRIMARY_REPO) libaio-aarch64 || \
+	git clone $(GIT_CLONE_OPTS) --branch $(LIBAIO_VERSION) $(FALLBACK_REPO) libaio-aarch64
 else
 	cd libaio-aarch64 && git fetch --tags && git checkout $(LIBAIO_VERSION)
 endif
@@ -348,7 +350,7 @@ else
 	cd processwatch && git fetch --tags && git checkout $(PROCESSWATCH_VERSION)
 endif
 	cd processwatch && ./build.sh
-	mkdir -p bin
+	mkdir -p bin/x86_64
 	cp processwatch/processwatch bin/x86_64/
 	strip --strip-unneeded bin/x86_64/processwatch
 
@@ -437,14 +439,18 @@ endif
 tsc:
 	cd tsc && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build
 
-TURBOSTAT_VERSION := 6.9.12
+# turbostat version 2025.12.02 is in kernel 6.19
+TURBOSTAT_LINUX_VERSION := 6.19.9
 turbostat:
 ifeq ("$(wildcard linux_turbostat)","")
-	wget $(WGET_OPTS) https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$(TURBOSTAT_VERSION).tar.xz
-	tar -xf linux-$(TURBOSTAT_VERSION).tar.xz && mv linux-$(TURBOSTAT_VERSION)/ linux_turbostat/
+	wget $(WGET_OPTS) https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$(TURBOSTAT_LINUX_VERSION).tar.xz
+	tar -xf linux-$(TURBOSTAT_LINUX_VERSION).tar.xz && mv linux-$(TURBOSTAT_LINUX_VERSION)/ linux_turbostat/
 endif
 	sed -i '/_Static_assert/d' linux_turbostat/tools/power/x86/turbostat/turbostat.c
 	cd linux_turbostat/tools/power/x86/turbostat && make
+	mkdir -p bin/x86_64
+	cp linux_turbostat/tools/power/x86/turbostat/turbostat bin/x86_64/
+	strip --strip-unneeded bin/x86_64/turbostat
 
 reset:
 	cd async-profiler
@@ -462,7 +468,6 @@ reset:
 	cd stress-ng && git clean -fdx && git reset --hard
 	cd sysstat && git clean -fdx && git reset --hard
 	cd tsc && rm -f tsc
-	cd linux_turbostat/tools/power/x86/turbostat && make clean
 
 # libs are not directly used in the build but are required in the oss archive file because some of the tools are statically linked
 # glibc 2.27 was the version used in Ubuntu 18.04 which is the base docker image we used to build the tools
@@ -475,5 +480,5 @@ libcrypt.tar.gz:
 libs: glibc.tar.gz zlib.tar.gz libcrypt.tar.gz
 
 oss-source: reset libs
-	tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ perf-archive/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ linux_turbostat/tools/power/x86/turbostat glibc.tar.gz zlib.tar.gz libcrypt.tar.gz
+	tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ perf-archive/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ glibc.tar.gz zlib.tar.gz libcrypt.tar.gz
 	md5sum oss_source.tgz > oss_source.tgz.md5

tools/build.Dockerfile

@@ -167,6 +167,7 @@ ADD . /workdir
 WORKDIR /workdir
 RUN make perf
 RUN make processwatch
+RUN make turbostat
 
 FROM scratch AS output
 COPY --from=builder workdir/bin /bin