From 88775c3299654a93bb940b3a6d3e59fbef7c32a3 Mon Sep 17 00:00:00 2001 From: Erwan Velu Date: Thu, 9 Jul 2026 07:34:18 +0200 Subject: [PATCH 1/5] hwgraph: add a Y midline between ticks on smp_scaling graphs Reading a value off the smp_scaling line graphs was imprecise with only the major Y ticks. Add a single intermediate horizontal gridline between two Y ticks (AutoMinorLocator(2)), a bit bolder than the default minor grid so it is actually readable, but still lighter than the solid major gridlines. Applies to the scaling line graphs (perf, perf_watt, watts, cpu_clock, cpu_ipc); the delta heatmaps are unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Erwan Velu --- graph/scaling.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/graph/scaling.py b/graph/scaling.py index 0c71b71..151078d 100644 --- a/graph/scaling.py +++ b/graph/scaling.py @@ -6,6 +6,7 @@ from matplotlib.colors import LinearSegmentedColormap, Normalize from matplotlib.lines import Line2D from matplotlib.offsetbox import AnchoredOffsetbox, HPacker, TextArea +from matplotlib.ticker import AutoMinorLocator from graph.graph import GRAPH_TYPES, Graph, numa_aggregated_components, numa_core_blocks, statistics_in_label from hwbench.bench.monitoring_structs import MonitoringContextKeys @@ -461,6 +462,10 @@ def smp_scaling_graph(args, output_dir, job: str, traces_name: list) -> int: ) graph.prepare_axes(8, 4) + # Add a midline between Y ticks to ease value reading (a bit more + # visible than the default minor grid, but still lighter than the major one). + graph.get_ax().yaxis.set_minor_locator(AutoMinorLocator(2)) + graph.get_ax().grid(which="minor", axis="y", linewidth=0.6, linestyle="dashed", color="0.6") graph.render() rendered_graphs += 1 From bb77e1af4f60017c392dfcaaf1d9be245ac13a43 Mon Sep 17 00:00:00 2001 From: Erwan Velu Date: Thu, 9 Jul 2026 07:42:53 +0200 Subject: [PATCH 2/5] hwgraph: keep the smp_scaling legend table aligned across traces The per-trace legend of the smp_scaling graphs shows, for each trace, " [min; mean; stddev; max]". Each label was padded to its own length, so traces with different name lengths produced a misaligned table where the [min; mean; stddev; max] columns did not line up, making the values harder to read and compare between traces. Pad the trace name and the values to a width common to all the traces of the graph (like the environment graphs already do), so the columns stay aligned whatever the trace names. Co-Authored-By: Claude Opus 4.8 (1M context) --- graph/scaling.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/graph/scaling.py b/graph/scaling.py index 151078d..bde0127 100644 --- a/graph/scaling.py +++ b/graph/scaling.py @@ -8,7 +8,14 @@ from matplotlib.offsetbox import AnchoredOffsetbox, HPacker, TextArea from matplotlib.ticker import AutoMinorLocator -from graph.graph import GRAPH_TYPES, Graph, numa_aggregated_components, numa_core_blocks, statistics_in_label +from graph.graph import ( + GRAPH_TYPES, + Graph, + get_max_value_string_length, + numa_aggregated_components, + numa_core_blocks, + statistics_in_label, +) from hwbench.bench.monitoring_structs import MonitoringContextKeys @@ -430,6 +437,17 @@ def smp_scaling_graph(args, output_dir, job: str, traces_name: list) -> int: "tab:brown", "tab:grey", ] + # Pad the trace name and the values to a common width so the + # per-trace legend table stays aligned even when trace names + # (or values) have different lengths. + max_title_length = max((len(trace_name) for trace_name in v_y_source[perf]), default=0) + max_value_length = max( + ( + get_max_value_string_length(np.array(v_y_source[perf][trace_name])) + for trace_name in v_y_source[perf] + ), + default=0, + ) # Traces are not ordered by growing cpu cores count # We need to prepare the x_serie to be sorted this way # The y_serie depends on the graph type @@ -439,7 +457,7 @@ def smp_scaling_graph(args, output_dir, job: str, traces_name: list) -> int: order = np.argsort(workers[trace_name]) x_serie = np.array(workers[trace_name])[order] y_serie = np.array(v_y_source[perf][trace_name])[order] - series_label = statistics_in_label(trace_name, y_serie) + series_label = statistics_in_label(trace_name, y_serie, max_title_length, max_value_length) # If we have an error distribution, let's use errorbars if v_err_source is not None: graph.get_ax().errorbar( From 3eb061ad909b4225e1788d9971ea4b466d48bea3 Mon Sep 17 00:00:00 2001 From: Erwan Velu Date: Thu, 9 Jul 2026 11:04:09 +0200 Subject: [PATCH 3/5] hwgraph: add per-core steady-state distribution graphs For every per-core CPU metric (Core frequency, Core IPC and CPU Core power consumption), render an additional graph showing how that metric is distributed across the cores at steady state. Each core's mean value over the run becomes one data point; a violin shows the density and an overlaid box shows the median (red), mean (green dashed), quartiles and outliers. Where the line graphs answer "how did the metric evolve over time?", these answer "how uniform were the cores?" -- immediately exposing stragglers, bimodal behaviour (e.g. boosted vs throttled cores) and the spread that a single averaged curve hides. Like the other per-core graphs they are rendered twice, once over all the cores (all_cores) and once restricted to the cores pinned during the job (pinned_cores), and land in the same per-metric directories next to their line graphs. The Y axis is autoscaled rather than zero-based: a distribution is unreadable squished against a zero baseline. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Erwan Velu --- graph/graph.py | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ graph/hwgraph.py | 13 +++++++++ 2 files changed, 86 insertions(+) diff --git a/graph/graph.py b/graph/graph.py index 9c78ea7..bfdd706 100644 --- a/graph/graph.py +++ b/graph/graph.py @@ -534,6 +534,79 @@ def numa_performance_heatmap( return 1 +def cpu_distribution_graph( + args, + output_dir, + bench: Bench, + component_type: MonitoringContextKeys, + item_title: str, + dir_suffix=None, + names=None, + title_note=None, +) -> int: + """Render the per-core distribution of a metric at steady state (violin + box). + + Each core's steady-state value (mean over the run) becomes one point; the + violin shows their density and the box shows median/mean/quartiles/outliers, + so the core-to-core uniformity is visible at a glance. names restricts the + cores (pinned view). The Y axis autoscales to the data (a distribution is + unreadable squished against a zero baseline). + """ + components = bench.get_all_metrics(component_type, "Core", names) + if not components: + return 0 + unit = bench.get_metric_unit(component_type) + values = [float(np.mean(component.get_mean())) for component in components] + + trace = bench.get_trace() + title = f'{item_title} per-core distribution during "{bench.get_bench_name()}" benchmark job\n{args.title}\n\n Stressor: ' + title += f"{bench.workers()} x {bench.get_title_engine_name()} for {bench.duration()} seconds" + title += f"\n{bench.get_system_title()}" + graph_dir = output_dir.joinpath(f"{trace.get_name()}/{bench.get_bench_name()}/{component_type!s}") + if dir_suffix: + graph_dir = graph_dir.joinpath(dir_suffix) + graph = Graph( + args, + title, + "", + unit, + graph_dir, + f"{item_title} - distribution", + square=True, + show_source_file=trace, + title_note=title_note, + ) + ax = graph.get_ax() + parts = ax.violinplot([values], positions=[1], widths=0.7, showextrema=False) + for body in parts["bodies"]: + body.set_facecolor("tab:blue") + body.set_alpha(0.25) + ax.boxplot( + [values], + positions=[1], + widths=0.15, + showmeans=True, + meanline=True, + medianprops=dict(color="tab:red"), + meanprops=dict(color="tab:green", linestyle="--"), + flierprops=dict(marker=".", markersize=3, alpha=0.4), + ) + ax.set_xticks([1]) + ax.set_xticklabels([f"{len(values)} cores"]) + ax.grid(which="major", axis="y", linewidth=0.6, linestyle="dashed", color="0.7") + legend = ax.legend( + handles=[ + Line2D([], [], color="tab:red", label="median"), + Line2D([], [], color="tab:green", linestyle="--", label="mean"), + ], + loc="upper right", + fontsize=8, + ) + graph.needs_legend = False + graph.render(extra_legend=legend) + return 1 + + def generic_graph( args, output_dir, diff --git a/graph/hwgraph.py b/graph/hwgraph.py index 4fa5260..721b7dc 100755 --- a/graph/hwgraph.py +++ b/graph/hwgraph.py @@ -20,6 +20,7 @@ from graph.chassis import graph_chassis from graph.common import fatal from graph.graph import ( + cpu_distribution_graph, generic_graph, init_matplotlib, numa_aggregated_components, @@ -516,6 +517,18 @@ def graph_cpu(args, trace: Trace, bench_name: str, output_dir) -> int: dir_suffix=dir_suffix, title_note=title_note, ) + # Per-core metrics also get a steady-state distribution (violin + box). + if filter == "Core": + rendered_graphs += cpu_distribution_graph( + args, + output_dir, + bench, + metric, + graph_name, + dir_suffix=dir_suffix, + names=names, + title_note=title_note, + ) return rendered_graphs From e3a662d0353fce3c775372a8e32ad538c55cf3bd Mon Sep 17 00:00:00 2001 From: Erwan Velu Date: Thu, 9 Jul 2026 11:09:46 +0200 Subject: [PATCH 4/5] hwgraph: add per-core distribution graphs to smp_scaling Mirror the per-core steady-state distribution graphs on the SMP scaling views. For each trace and each per-core metric (Core frequency, Core IPC and CPU Core power consumption), one graph plots a violin + box per scaling step (X = worker count, evenly spaced), showing how the core-to-core distribution of that metric evolves as the sweep grows. The scaling line graphs plot a single averaged value per step; these expose the spread that average hides -- cores that start to diverge or throttle only past a given worker count, bimodal frequency behaviour, or a widening per-core power spread. The box reports median (red), mean (green dashed), quartiles and outliers. Like the other per-core scaling graphs they are rendered for all_cores and, when the sweep pins cores, pinned_cores, landing in the cpu_clock, cpu_ipc and cpu_core_power directories next to the matching line graphs. The Y axis is autoscaled rather than zero-based: a distribution is unreadable squished against a zero baseline. Why violin graphs at both levels: the per-benchmark (steady-state) and the per-scaling-step distributions answer two complementary questions that averages and time-series curves both flatten. - The individual, steady-state violin answers "for this one operating point, how uniform is the hardware?". A tight body means every core behaves alike; a long tail or a second lobe exposes a straggler, an asymmetric NUMA/boost domain or a mis-pinned core -- something a single mean value, and even the time-series line graph, hides because they collapse every core into one number per instant. - The scaling violins put those same distributions side by side along the load axis, so the shape itself becomes the signal: you watch the spread widen or split exactly at the worker count where cores begin to contend for power/thermal budget, and you can tell a uniform slow-down (body drops as a block) from a divergence (body stretches or splits) -- a distinction a scaling line of per-step averages cannot make. Together they let a reader first confirm a single point is healthy, then follow how that health degrades under increasing load, using the same visual vocabulary at both zoom levels. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Erwan Velu --- graph/scaling.py | 101 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/graph/scaling.py b/graph/scaling.py index bde0127..787dcd2 100644 --- a/graph/scaling.py +++ b/graph/scaling.py @@ -208,6 +208,104 @@ def render_numa_delta_heatmaps(args, temp_outdir, job: str, emp: str) -> int: return rendered +def render_scaling_distributions(args, temp_outdir, job: str, emp: str, has_ipc: bool) -> int: + """Render the per-core metric distribution across the scaling steps. + + For each trace and each per-core metric (frequency, IPC and per-core power), + one graph shows how the core-to-core distribution evolves as the sweep grows: + X = worker count (one violin + box per scaling step, evenly spaced), Y = the + metric. The violin shows the density across cores, the box the median (red), + mean (green dashed), quartiles and outliers. + + Where the scaling line graphs plot a single averaged value per step, these + expose the spread hidden behind that average -- e.g. cores starting to + throttle or diverge only past a given worker count. Like the other per-core + scaling graphs they are rendered for all_cores and, when the sweep pins + cores, pinned_cores, and the Y axis is autoscaled (a distribution is + unreadable squished against a zero baseline). + """ + metrics = [ + ("cpu_clock", MonitoringContextKeys.Freq, "Mhz", "Core frequency"), + ] # type: list + if has_ipc: + metrics.append(("cpu_ipc", MonitoringContextKeys.IPC, "IPC", "Core IPC")) + metrics.append(("cpu_core_power", MonitoringContextKeys.PowerConsumption, None, "CPU Core power consumption")) + + rendered = 0 + for trace in args.traces: + trace_benches = trace.get_benches_by_job_per_emp(job) + if emp not in trace_benches or not trace_benches[emp]["bench"]: + continue + benches = trace_benches[emp]["bench"] + engine = benches[0].get_title_engine_name().replace(" ", "_") + trace_slug = trace.get_name().replace(" ", "_").replace("/", "_") + # all_cores always; pinned_cores when at least one run of the sweep pins cores. + variants = [("all_cores", False, None)] # type: list + if any(bench.cpu_pin() for bench in benches): + variants.append(("pinned_cores", True, "View limited to the pinned cores of each scaling step")) + + for dirname, context, unit, item_title in metrics: + for dir_suffix, use_pinned, note in variants: + data = [] # type: list + labels = [] # type: list + metric_unit = "" + for bench in sorted(benches, key=lambda b: b.workers()): + names = bench.pinned_core_names() if use_pinned else None + components = bench.get_all_metrics(context, "Core", names) + if not components: + continue + data.append([float(np.mean(component.get_mean())) for component in components]) + labels.append(bench.workers()) + metric_unit = bench.get_metric_unit(context) + if not data: + continue + + title = f'{args.title}\n\n{item_title} per-core distribution scaling via "{job}" benchmark job\n\n Stressor: ' + title += f"{benches[0].get_title_engine_name()} for {benches[0].duration()} seconds" + graph = Graph( + args, + title, + "Workers (scaling step)", + unit or metric_unit, + temp_outdir.joinpath(dirname, dir_suffix), + f"scaling_{dirname}_distribution_{trace_slug}_{engine}", + square=True, + show_source_file=trace, + title_note=note, + ) + ax = graph.get_ax() + positions = list(range(1, len(data) + 1)) + parts = ax.violinplot(data, positions=positions, widths=0.7, showextrema=False) + for body in parts["bodies"]: + body.set_facecolor("tab:blue") + body.set_alpha(0.25) + ax.boxplot( + data, + positions=positions, + widths=0.15, + showmeans=True, + meanline=True, + medianprops=dict(color="tab:red"), + meanprops=dict(color="tab:green", linestyle="--"), + flierprops=dict(marker=".", markersize=3, alpha=0.4), + ) + ax.set_xticks(positions) + ax.set_xticklabels(labels) + ax.grid(which="major", axis="y", linewidth=0.6, linestyle="dashed", color="0.7") + legend = ax.legend( + handles=[ + Line2D([], [], color="tab:red", label="median"), + Line2D([], [], color="tab:green", linestyle="--", label="mean"), + ], + loc="upper right", + fontsize=8, + ) + graph.needs_legend = False + graph.render(extra_legend=legend) + rendered += 1 + return rendered + + def smp_scaling_graph(args, output_dir, job: str, traces_name: list) -> int: """Render line graphs to compare performance scaling.""" rendered_graphs = 0 @@ -490,4 +588,7 @@ def smp_scaling_graph(args, output_dir, job: str, traces_name: list) -> int: # Per-NUMA-domain delta heatmap comparing the two traces (reference vs other). rendered_graphs += render_numa_delta_heatmaps(args, temp_outdir, job, emp) + # Per-core distribution (violin + box) across the scaling steps. + rendered_graphs += render_scaling_distributions(args, temp_outdir, job, emp, has_ipc) + return rendered_graphs From cd9adbf65b357df7e8ad03eaa9dde52f774fcf5f Mon Sep 17 00:00:00 2001 From: Erwan Velu Date: Thu, 9 Jul 2026 20:21:20 +0200 Subject: [PATCH 5/5] hwgraph: add per-NUMA-domain distribution graphs Add a distribution view for per-core CPU metrics (frequency, IPC, core power) aggregated by NUMA domain, complementing the existing per-domain line graphs and heatmaps: - numa_distribution_graph: one violin + box per NUMA domain for a single benchmark job (steady-state), landing next to the matching line graph in the same all_numa/pinned_numa directories. - render_numa_scaling_ridgelines: the SMP-scaling counterpart. A first grouped-violin attempt (one violin per domain per step, all in one axes) became unreadable past a handful of domains and steps. After reviewing sample renderings of several alternatives (median+IQR line, small multiples, spread heatmap, ridgeline), the ridgeline design was picked: one panel per scaling step -- every step, laid out as a grid so the figure grows in rows rather than becoming unreadable -- each panel a stacked density (ridgeline) per NUMA domain, preserving the full distribution shape (skew, bimodality) that a single averaged value would flatten. The grid's header/footer are reserved as a constant number of inches (not a fraction of the figure) so they keep the same size regardless of how many rows a long sweep needs. Both are rendered for all_numa (every core of each domain) and, when the sweep pins cores, pinned_numa. Also mark which CPU package each NUMA domain belongs to: hwbench does not record this directly, but same-package domains are always much closer to each other in the NUMA distance matrix than cross-package ones, so _numa_domains_by_package groups domains whose mutual distance is below a fixed threshold (union-find), with no need to know the socket count up front. Each scaling ridgeline panel gets a pale background wash per package behind its ridges, with a "CPU package " legend, colour-matched to the washes, centered below the whole grid. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Erwan Velu --- graph/graph.py | 112 ++++++++++++++- graph/hwgraph.py | 13 ++ graph/scaling.py | 355 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 477 insertions(+), 3 deletions(-) diff --git a/graph/graph.py b/graph/graph.py index bfdd706..493cf14 100644 --- a/graph/graph.py +++ b/graph/graph.py @@ -248,12 +248,17 @@ def trace_events(self): def render(self, extra_legend=None): """Render the graph to a file. - extra_legend: an additional, manually placed legend that must be taken - into account when computing the tight bounding box. + extra_legend: one or more additional, manually placed legends that + must be taken into account when computing the tight bounding box. """ # Retrieve the rendering file format file_format = self.args.format - legends = [extra_legend] if extra_legend else [] + if extra_legend is None: + legends = [] + elif isinstance(extra_legend, (list, tuple)): + legends = list(extra_legend) + else: + legends = [extra_legend] # Trace the events passed on the command line self.trace_events() @@ -460,6 +465,107 @@ def numa_distance_heatmap(args, output_dir, trace) -> int: return 1 +def numa_distribution_graph( + args, + output_dir, + bench: Bench, + component_type: MonitoringContextKeys, + item_title: str, + numa_nodes, + dir_suffix=None, + pinned_cores=None, + title_note=None, +) -> int: + """Render the per-NUMA-domain distribution of a metric at steady state (one violin + box per domain). + + Each domain gets its own violin built from its cores' steady-state (mean + over the run) values, so core-to-core uniformity can be compared across + domains at a glance -- same idea as cpu_distribution_graph, split by NUMA + domain instead of blended into a single body. When pinned_cores is given, + only the pinned cores of each domain feed its violin and domains with no + pinned core are skipped. The Y axis autoscales to the data. + """ + domains = [] + for node in sorted(numa_nodes): + core_names = {f"Core_{cpu}" for cpu in numa_nodes[node]} + if pinned_cores is not None: + core_names &= pinned_cores + if not core_names: + continue + cores = bench.get_all_metrics(component_type, names=core_names) + if not cores: + continue + values = [float(np.mean(core.get_mean())) for core in cores] + domains.append((node, values)) + if not domains: + return 0 + unit = bench.get_metric_unit(component_type) + + trace = bench.get_trace() + title = f'{item_title} per-NUMA-domain distribution during "{bench.get_bench_name()}" benchmark job\n{args.title}\n\n Stressor: ' + title += f"{bench.workers()} x {bench.get_title_engine_name()} for {bench.duration()} seconds" + title += f"\n{bench.get_system_title()}" + graph_dir = output_dir.joinpath(f"{trace.get_name()}/{bench.get_bench_name()}/{component_type!s}") + if dir_suffix: + graph_dir = graph_dir.joinpath(dir_suffix) + graph = Graph( + args, + title, + "", + unit, + graph_dir, + f"{item_title} - distribution", + show_source_file=trace, + title_note=title_note, + ) + ax = graph.get_ax() + positions = list(range(1, len(domains) + 1)) + domain_values = [values for _, values in domains] + parts = ax.violinplot(domain_values, positions=positions, widths=0.7, showextrema=False) + for body in parts["bodies"]: + body.set_facecolor("tab:blue") + body.set_alpha(0.25) + ax.boxplot( + domain_values, + positions=positions, + widths=0.15, + showmeans=True, + meanline=True, + medianprops=dict(color="tab:red"), + meanprops=dict(color="tab:green", linestyle="--"), + flierprops=dict(marker=".", markersize=3, alpha=0.4), + ) + ax.set_xticks(positions) + ax.set_xticklabels([f"NUMA {node}" for node, _ in domains]) + ax.grid(which="major", axis="y", linewidth=0.6, linestyle="dashed", color="0.7") + + # Left-side box listing each domain's cores in condensed form, like the + # heatmap's legend -- useful to see exactly which cores fed each violin, + # especially on the pinned view where a domain may keep only a few. + pinned_cpus = None + if pinned_cores is not None: + pinned_cpus = {int(name.split("_")[1]) for name in pinned_cores} + node_cores = [] + for node, _ in domains: + cpus = numa_nodes[node] + if pinned_cpus is not None: + cpus = [cpu for cpu in cpus if cpu in pinned_cpus] + node_cores.append((node, cpus)) + cores_legend = numa_cores_legend(ax, node_cores) + stats_legend = ax.legend( + handles=[ + Line2D([], [], color="tab:red", label="median"), + Line2D([], [], color="tab:green", linestyle="--", label="mean"), + ], + loc="upper right", + fontsize=8, + ) + ax.add_artist(cores_legend) + graph.needs_legend = False + graph.render(extra_legend=[cores_legend, stats_legend]) + return 1 + + def numa_performance_heatmap( args, output_dir, diff --git a/graph/hwgraph.py b/graph/hwgraph.py index 721b7dc..ccca8d8 100755 --- a/graph/hwgraph.py +++ b/graph/hwgraph.py @@ -25,6 +25,7 @@ init_matplotlib, numa_aggregated_components, numa_distance_heatmap, + numa_distribution_graph, numa_performance_heatmap, yerr_graph, ) @@ -589,6 +590,18 @@ def graph_cpu_numa(args, trace: Trace, bench_name: str, output_dir) -> int: pinned_cores=pinned_cores, title_note=title_note, ) + # Companion violin: one steady-state distribution per NUMA domain. + rendered_graphs += numa_distribution_graph( + args, + output_dir, + bench, + metric, + graph_name, + numa_nodes, + dir_suffix=dir_suffix, + pinned_cores=pinned_cores, + title_note=title_note, + ) return rendered_graphs diff --git a/graph/scaling.py b/graph/scaling.py index 787dcd2..e248f44 100644 --- a/graph/scaling.py +++ b/graph/scaling.py @@ -1,11 +1,16 @@ +from __future__ import annotations + from itertools import cycle from statistics import stdev from typing import Any # noqa: F401 +import matplotlib +import matplotlib.pyplot as plt import numpy as np from matplotlib.colors import LinearSegmentedColormap, Normalize from matplotlib.lines import Line2D from matplotlib.offsetbox import AnchoredOffsetbox, HPacker, TextArea +from matplotlib.patches import Patch from matplotlib.ticker import AutoMinorLocator from graph.graph import ( @@ -306,6 +311,352 @@ def render_scaling_distributions(args, temp_outdir, job: str, emp: str, has_ipc: return rendered +# Colours cycled across NUMA domains in the render_numa_scaling_* graphs below, so +# each domain keeps the same colour everywhere it appears. tab20 gives 20 distinct +# colours, enough for every domain count seen on real systems so far; it only wraps +# (and two domains start sharing a colour) past that. +_NUMA_DOMAIN_COLORS = [matplotlib.colormaps["tab20"](i) for i in range(20)] + +# Pale background washes used by render_numa_scaling_ridgelines to mark which CPU +# package each NUMA domain belongs to, cycled per package. Kept very light so they +# stay subordinate to the (much more saturated) per-domain ridge colours. +_NUMA_PACKAGE_BAND_COLORS = ["#eaf1fb", "#fbf1e2", "#eaf7ee", "#f6eef8"] + + +# hwbench does not record which package a NUMA domain sits on, but same-package +# domains are always much closer to each other than cross-package ones (see the +# sample matrices in hwbench/environment/numa.py: ~10-12 within a package, 32+ +# across). 20 sits comfortably between the two tiers. +_NUMA_SAME_PACKAGE_DISTANCE = 20 + + +def _numa_domains_by_package(numa_nodes: dict, numa_distances: dict) -> dict | None: + """Group NUMA domains into CPU packages, from the distance matrix. + + Connects domains whose mutual distance is below _NUMA_SAME_PACKAGE_DISTANCE + (union-find) and returns {domain: package index}. Returns None when there is + nothing to distinguish (everything lands in one group, e.g. a single socket) + or the matrix is missing/incomplete. + """ + if len(numa_nodes) <= 1 or not numa_distances: + return None + domains = sorted(numa_nodes) + if not all(d in numa_distances and len(numa_distances[d]) > max(domains) for d in domains): + return None + + parent = {d: d for d in domains} + + def find(node): + while parent[node] != node: + parent[node] = parent[parent[node]] + node = parent[node] + return node + + for d in domains: + for o in domains: + if d != o and numa_distances[d][o] < _NUMA_SAME_PACKAGE_DISTANCE: + root_d, root_o = find(d), find(o) + if root_d != root_o: + parent[root_d] = root_o + + groups: dict = {} + for d in domains: + groups.setdefault(find(d), []).append(d) + + if len(groups) <= 1: + return None + ordered_groups = sorted(groups.values(), key=min) + return {d: package for package, group in enumerate(ordered_groups) for d in group} + + +def _numa_scaling_metrics(has_ipc: bool) -> list: + """Metrics rendered by the render_numa_scaling_* graphs below.""" + metrics = [ + ("cpu_clock", MonitoringContextKeys.Freq, "Mhz", "Core frequency per NUMA domain"), + ] # type: list + if has_ipc: + metrics.append(("cpu_ipc", MonitoringContextKeys.IPC, "IPC", "Core IPC per NUMA domain")) + metrics.append( + ( + "cpu_core_power", + MonitoringContextKeys.PowerConsumption, + None, + "CPU Core power consumption per NUMA domain", + ) + ) + return metrics + + +def _numa_scaling_variants(benches: list) -> list: + """all_numa always; pinned_numa when at least one run of the sweep pins cores.""" + variants = [("all_numa", False, None)] # type: list + if any(bench.cpu_pin() for bench in benches): + variants.append(("pinned_numa", True, "View limited to the pinned logical cores of each scaling step")) + return variants + + +def _collect_numa_step_domain_values(benches: list, context, numa_nodes, use_pinned: bool) -> dict: + """Return {worker count: {domain: [per-core steady-state values]}} for one metric/variant.""" + per_step = {} # type: dict[int, dict[int, list]] + for bench in benches: + pinned = bench.pinned_core_names() if use_pinned else None + step_domains = {} + for node in sorted(numa_nodes): + core_names = {f"Core_{cpu}" for cpu in numa_nodes[node]} + if pinned is not None: + core_names &= pinned + if not core_names: + continue + cores = bench.get_all_metrics(context, names=core_names) + if not cores: + continue + step_domains[node] = [float(np.mean(core.get_mean())) for core in cores] + if step_domains: + per_step[bench.workers()] = step_domains + return per_step + + +def _numa_scaling_cores_by_domain(numa_nodes, domains: list, benches: list, use_pinned: bool) -> dict: + """{domain: [cpus]}, restricted to the union of cores pinned across the sweep on the pinned view.""" + pinned_union = None # type: set | None + if use_pinned: + pinned_union = set() + for bench in benches: + pinned_union |= {int(name.split("_")[1]) for name in (bench.pinned_core_names() or set())} + cores_by_domain = {} + for node in domains: + cpus = numa_nodes[node] + if pinned_union is not None: + cpus = [cpu for cpu in cpus if cpu in pinned_union] + cores_by_domain[node] = cpus + return cores_by_domain + + +def _save_standalone_figure(args, fig, output_dir, filename: str) -> None: + """Save a figure built without the Graph wrapper (used by grid/multi-axes graphs). + + Mirrors Graph.render()'s save call so standalone figures match the rest of + hwgraph's output (same directory creation, format, dpi and tight bbox). + """ + output_dir.mkdir(parents=True, exist_ok=True) + fig.savefig( + f"{output_dir}/{filename}.{args.format}", + format=args.format, + dpi=args.dpi, + bbox_inches="tight", + pad_inches=1, + ) + fig.clear() + plt.close(fig) + + +def _gaussian_kde_curve(values: list, sample_points: np.ndarray) -> np.ndarray: + """Scipy-free Gaussian KDE (Silverman's rule-of-thumb bandwidth), unnormalized.""" + values_arr = np.asarray(values, dtype=float) + std = float(np.std(values_arr)) + bandwidth = 1.06 * std * len(values_arr) ** (-1 / 5) if std > 0 else 1.0 + bandwidth = max(bandwidth, 1e-3) + diffs = (sample_points[:, None] - values_arr[None, :]) / bandwidth + return np.exp(-0.5 * diffs**2).sum(axis=1) + + +# render_numa_scaling_ridgelines lays out one panel per step, _RIDGELINE_MAX_COLS +# per row, so the grid (and the figure) grows taller as a sweep has more steps. +# The header/footer are reserved as a constant number of INCHES rather than a +# fraction of the figure, so they keep the same size and never shrink to +# nothing (or balloon) as the number of rows changes. +_RIDGELINE_PANEL_WIDTH = 3.0 +_RIDGELINE_PANEL_HEIGHT = 3.3 +_RIDGELINE_MAX_COLS = 5 +_RIDGELINE_SUPTITLE_INSET = 0.05 # inches from the top edge to the suptitle anchor +_RIDGELINE_NOTE_INSET = 1.3 # inches from the top edge to the note (when present) +_RIDGELINE_HEADER_INCHES = 1.6 # reserved header height: title only +_RIDGELINE_HEADER_INCHES_NOTE = 2.0 # reserved header height: title + note +_RIDGELINE_FOOTER_INCHES = 0.5 # reserved footer height: source caption +_RIDGELINE_FOOTER_INCHES_PACKAGE = 0.85 # reserved footer height: source caption + package legend +_RIDGELINE_CAPTION_INSET = 0.08 # inches from the bottom edge to the source caption +_RIDGELINE_PACKAGE_LEGEND_INSET = 0.4 # inches from the bottom edge to the package legend + + +def render_numa_scaling_ridgelines(args, temp_outdir, job: str, emp: str, has_ipc: bool) -> int: + """Render the per-NUMA-domain metric distribution at every scaling step. + + For each trace and each per-NUMA metric (frequency, IPC and per-core + power, aggregated by domain), one graph lays out one ridgeline panel per + scaling step -- every step, not a sample -- in a grid: each panel is a + stacked density (ridgeline) per NUMA domain, preserving the full + distribution shape (skew, bimodality) that a single averaged value would + flatten. + + Like the other per-domain scaling graphs, rendered for all_numa (every + core of each domain) and, when the sweep pins cores, pinned_numa. Needs + the NUMA topology in each trace; older traces without it are skipped. + """ + rendered = 0 + for trace in args.traces: + # Topology is per-host: read it from this trace, not a shared reference, + # so hosts with different NUMA-node counts each render their own domains. + numa_nodes = trace.get_numa_nodes() + if not numa_nodes: + continue + # Best-effort: which package each domain sits on, so multi-socket systems get a + # background wash marking the boundary; None on a single socket or when the + # distance matrix doesn't cleanly cluster (older trace, unexpected encoding). + domain_package = _numa_domains_by_package(numa_nodes, trace.get_numa_distances()) + + trace_benches = trace.get_benches_by_job_per_emp(job) + if emp not in trace_benches or not trace_benches[emp]["bench"]: + continue + benches = sorted(trace_benches[emp]["bench"], key=lambda b: b.workers()) + engine = benches[0].get_title_engine_name().replace(" ", "_") + trace_slug = trace.get_name().replace(" ", "_").replace("/", "_") + variants = _numa_scaling_variants(benches) + + for dirname, context, unit, item_title in _numa_scaling_metrics(has_ipc): + for dir_suffix, use_pinned, note in variants: + per_step = _collect_numa_step_domain_values(benches, context, numa_nodes, use_pinned) + if not per_step: + continue + steps = sorted(per_step) + domains = sorted({node for step_domains in per_step.values() for node in step_domains}) + metric_unit = unit or benches[0].get_metric_unit(context) + cores_by_domain = _numa_scaling_cores_by_domain(numa_nodes, domains, benches, use_pinned) + domain_color = { + node: _NUMA_DOMAIN_COLORS[i % len(_NUMA_DOMAIN_COLORS)] for i, node in enumerate(domains) + } + + all_values = [ + v for step_domains in per_step.values() for values in step_domains.values() for v in values + ] + pad = (max(all_values) - min(all_values)) * 0.05 or 1 + yy = np.linspace(min(all_values) - pad, max(all_values) + pad, 200) + + # Every step gets a panel -- no sampling -- laid out as a grid so the + # figure grows in rows rather than becoming arbitrarily wide. + cols = min(_RIDGELINE_MAX_COLS, len(steps)) + rows = -(-len(steps) // cols) # ceil division + header_inches = _RIDGELINE_HEADER_INCHES_NOTE if note else _RIDGELINE_HEADER_INCHES + footer_inches = ( + _RIDGELINE_FOOTER_INCHES_PACKAGE if domain_package is not None else _RIDGELINE_FOOTER_INCHES + ) + fig_width = _RIDGELINE_PANEL_WIDTH * cols + fig_height = _RIDGELINE_PANEL_HEIGHT * rows + header_inches + footer_inches + fig, axes = plt.subplots(rows, cols, figsize=(fig_width, fig_height), sharey=True, squeeze=False) + flat_axes = axes.flatten() + + for ax, step in zip(flat_axes, steps): + step_domains = per_step[step] + present = [node for node in domains if node in step_domains] + offsets = np.arange(len(present)) * 1.1 + + # Background wash per CPU package, drawn first so the ridges sit on top. + if domain_package is not None: + package_offsets: dict = {} + for offset, node in zip(offsets, present): + package_offsets.setdefault(domain_package[node], []).append(offset) + for package, offs in package_offsets.items(): + ax.axvspan( + min(offs) - 0.55, + max(offs) + 0.55, + color=_NUMA_PACKAGE_BAND_COLORS[package % len(_NUMA_PACKAGE_BAND_COLORS)], + zorder=0, + ) + + for offset, node in zip(offsets, present): + density = _gaussian_kde_curve(step_domains[node], yy) + peak = density.max() + if peak > 0: + density = density / peak + ax.fill_betweenx(yy, offset, offset + density, color=domain_color[node], alpha=0.8, zorder=2) + ax.plot(offset + density, yy, color="black", linewidth=0.5, zorder=2) + ax.set_xticks(offsets) + ax.set_xticklabels([f"N{node}" for node in present], rotation=90, fontsize=6) + ax.set_title(f"{step} workers", fontsize=9) + ax.set_xlim(-0.3, (offsets[-1] if len(offsets) else 0) + 1.3) + ax.grid(which="major", axis="y", linewidth=0.4, linestyle="dashed", color="0.85", zorder=1) + for ax in flat_axes[len(steps) :]: + ax.axis("off") + # Y label on the leftmost panel of every row (sharey makes the scale common). + for row in range(rows): + flat_axes[row * cols].set_ylabel(metric_unit) + + node_width = max((len(str(node)) for node in domains), default=1) + handles = [Line2D([], [], color=domain_color[node], linewidth=6) for node in domains] + labels = [ + f"NUMA {node:>{node_width}}" + + (f" (pkg {domain_package[node]})" if domain_package is not None else "") + + f": {numa_core_blocks(cores_by_domain[node])}" + for node in domains + ] + fig.legend( + handles, + labels, + loc="center left", + bbox_to_anchor=(1.0, 0.5), + title="NUMA domain [cores]", + fontsize=8, + ) + # Separate legend explaining the background wash colour, centered below the + # grid (fig.legend keeps every call as its own legend, unlike ax.legend which + # replaces the last one). + if domain_package is not None: + package_count = len(set(domain_package.values())) + package_handles = [ + Patch( + facecolor=_NUMA_PACKAGE_BAND_COLORS[package % len(_NUMA_PACKAGE_BAND_COLORS)], + edgecolor="0.6", + ) + for package in range(package_count) + ] + package_labels = [f"CPU package {package}" for package in range(package_count)] + fig.legend( + package_handles, + package_labels, + loc="lower center", + bbox_to_anchor=(0.5, _RIDGELINE_PACKAGE_LEGEND_INSET / fig_height), + ncol=package_count, + fontsize=8, + ) + + title = f'{args.title}\n\n{item_title} distribution scaling via "{job}" benchmark job\n\n Stressor: ' + title += f"{benches[0].get_title_engine_name()} for {benches[0].duration()} seconds" + fig.suptitle(title, fontsize=11, y=1 - _RIDGELINE_SUPTITLE_INSET / fig_height) + if note: + fig.text( + 0.5, + 1 - _RIDGELINE_NOTE_INSET / fig_height, + note, + ha="center", + color="darkred", + fontweight="bold", + fontsize=10, + ) + fig.text( + 0.01, + _RIDGELINE_CAPTION_INSET / fig_height, + f"data plotted from {trace.get_filename()}", + fontsize=8, + bbox=dict(boxstyle="round", facecolor="white", alpha=0.6), + ) + fig.tight_layout( + rect=( + 0.02, + footer_inches / fig_height, + 1, + 1 - header_inches / fig_height, + ) + ) + + _save_standalone_figure( + args, + fig, + temp_outdir.joinpath(dirname, dir_suffix), + f"scaling_{dirname}_numa_ridgeline_{trace_slug}_{engine}", + ) + rendered += 1 + return rendered + + def smp_scaling_graph(args, output_dir, job: str, traces_name: list) -> int: """Render line graphs to compare performance scaling.""" rendered_graphs = 0 @@ -591,4 +942,8 @@ def smp_scaling_graph(args, output_dir, job: str, traces_name: list) -> int: # Per-core distribution (violin + box) across the scaling steps. rendered_graphs += render_scaling_distributions(args, temp_outdir, job, emp, has_ipc) + # Per-NUMA-domain distribution across the scaling steps: a ridgeline grid, + # one panel per step, one density per domain within each panel. + rendered_graphs += render_numa_scaling_ridgelines(args, temp_outdir, job, emp, has_ipc) + return rendered_graphs