diff --git a/graph/graph.py b/graph/graph.py index 3c8c0a6..bd0f3bc 100644 --- a/graph/graph.py +++ b/graph/graph.py @@ -6,12 +6,14 @@ import matplotlib.pyplot as plt import numpy as np from matplotlib.collections import LineCollection +from matplotlib.lines import Line2D from matplotlib.pylab import Axes from matplotlib.ticker import AutoMinorLocator, FuncFormatter, MultipleLocator from graph.common import fatal from graph.trace import Bench from hwbench.bench.monitoring_structs import MonitoringContextKeys, MonitorMetric +from hwbench.utils.helpers import cpu_list_to_range MEAN = "mean" ERROR = "error" @@ -243,11 +245,15 @@ def trace_events(self): color=event_color, ) - def render(self): - """Render the graph to a file.""" + def render(self, extra_legend=None): + """Render the graph to a file. + + extra_legend: an additional, manually placed legend that must be taken + into account when computing the tight bounding box. + """ # Retrieve the rendering file format file_format = self.args.format - legends = [] + legends = [extra_legend] if extra_legend else [] # Trace the events passed on the command line self.trace_events() @@ -309,6 +315,225 @@ def statistics_in_label(label: str, serie: np.ndarray, max_title_length=0, max_v return f"{label:{max_title_length}} [{fp(min(serie), max_value_length)}; {fp(np.mean(serie), max_value_length)}; {fp(np.std(serie), max_value_length)}; {fp(max(serie), max_value_length)}]" +def numa_core_blocks(cpus, width: int = 3) -> str: + """Render a cpu list as aligned, individually bracketed range blocks. + + Reuses cpu_list_to_range() and only reformats its output: each range block + gets its own "[]", the numbers are padded to `width` digits with the dash + centered, so blocks line up in a monospace legend. + e.g. [0..7, 64..71] -> "[0 - 7] [64 - 71]". + """ + blocks = [] + for block in cpu_list_to_range(list(cpus)).split(", "): + low, _, high = block.partition("-") + if high: + # Right-justify both bounds so numbers align and the dash stays centered. + blocks.append(f"[{low:>{width}}-{high:>{width}}]") + else: + # A single core (no range): keep the same block width, right-aligned. + blocks.append(f"[{block:>{2 * width + 1}}]") + return ", ".join(blocks) + + +def numa_aggregated_components( + bench: Bench, component_type: MonitoringContextKeys, numa_nodes, pinned_cores=None +) -> list[MonitorMetric]: + """Aggregate per-core metrics into one synthetic metric per NUMA domain. + + For each NUMA domain, average (per sample) the metric of the cores that + belong to it. When pinned_cores (a set of "Core_N" names) is given, only the + pinned cores of each domain are averaged and domains with no pinned core are + dropped. Domains with no monitored core are skipped. The result is a list of + MonitorMetric objects (one per domain) ready to feed generic_graph. + """ + samples_count = bench.get_samples_count() + components = [] + for node in sorted(numa_nodes): + core_names = {f"Core_{cpu}" for cpu in numa_nodes[node]} + if pinned_cores is not None: + core_names &= pinned_cores + if not core_names: + continue + cores = bench.get_all_metrics(component_type, names=core_names) + if not cores: + continue + means = [] + for sample in range(samples_count): + values = [c.get_mean()[sample] for c in cores if sample < len(c.get_mean())] + means.append(float(np.mean(values)) if values else 0.0) + metric = MonitorMetric(f"NUMA {node}", cores[0].get_unit()) + metric.mean = means + metric.full_name = f"NUMA {node}" + components.append(metric) + return components + + +def numa_cores_legend(ax, node_cores): + """Add a left-side box listing each NUMA domain's cores in condensed form. + + node_cores is an ordered list of (domain, cpus) pairs (top to bottom). The + box is anchored well to the left of the Y axis so it does not collide with + it, even with 3-digit core numbers. + """ + node_width = max((len(str(node)) for node, _ in node_cores), default=1) + labels = [f"NUMA {node:>{node_width}}: {numa_core_blocks(cpus)}" for node, cpus in node_cores] + handles = [Line2D([], [], linestyle="none") for _ in labels] + return ax.legend( + handles, + labels, + loc="upper right", + bbox_to_anchor=(-0.10, 1), + title="NUMA domain [cores]", + handlelength=0, + handletextpad=0, + ) + + +def _render_numa_heatmap(graph, nodes, matrix, extra_legend=None) -> None: + """Draw a NUMA domain x domain distance matrix onto graph. + + Cells are colored/annotated by the inter-domain distance. + """ + ax = graph.get_ax() + image = ax.imshow(matrix, cmap="viridis") + graph.fig.colorbar(image, ax=ax, fraction=0.046, pad=0.04, label="NUMA distance") + labels = [f"NUMA {node}" for node in nodes] + ax.set_xticks(range(len(nodes))) + ax.set_yticks(range(len(nodes))) + ax.set_xticklabels(labels) + ax.set_yticklabels(labels) + # Text color threshold so annotations stay readable over the colormap. + threshold = (matrix.max() + matrix.min()) / 2 + for i in range(len(nodes)): + for j in range(len(nodes)): + text, weight = f"{matrix[i, j]:.0f}", "normal" + ax.text( + j, + i, + text, + ha="center", + va="center", + fontsize=8, + fontweight=weight, + color="white" if matrix[i, j] < threshold else "black", + ) + graph.needs_legend = False + graph.render(extra_legend=extra_legend) + + +def numa_distance_heatmap(args, output_dir, trace) -> int: + """Render the host's NUMA distance matrix as a heatmap. + + This is a per-host topology figure (one per trace), independent of any + benchmark or performance metric: cell color/value is the distance between + two NUMA domains, so which domains are close/far is visible at a glance. + Needs the distance matrix with at least two domains; otherwise nothing is + rendered. + """ + distances = trace.get_numa_distances() + if not distances or len(distances) < 2: + return 0 + nodes = sorted(distances) + try: + matrix = np.array([[distances[src][dst] for dst in nodes] for src in nodes], dtype=float) + except (KeyError, IndexError): + return 0 + + cpu = trace.get_cpu() + dmi = trace.get_dmi() + title = f"NUMA distances of {trace.get_name()}\n{args.title}\n\n" + title += f"System: {dmi['serial']} {dmi['product']}" + title += f"\nProcessor: {cpu.get('sockets', 1)}x {cpu['model']} - {cpu['numa_domains']} NUMA domains" + graph = Graph( + args, + title, + "NUMA domain", + "NUMA domain", + output_dir.joinpath(trace.get_name()), + "NUMA distances", + square=True, + show_source_file=trace, + ) + numa_nodes = trace.get_numa_nodes() + legend = numa_cores_legend(graph.get_ax(), [(node, numa_nodes.get(node, [])) for node in nodes]) + _render_numa_heatmap(graph, nodes, matrix, extra_legend=legend) + return 1 + + +def numa_performance_heatmap( + args, + output_dir, + bench: Bench, + component_type: MonitoringContextKeys, + item_title: str, + numa_nodes, + dir_suffix=None, + pinned_cores=None, + title_note=None, +) -> int: + """Render a NUMA-domain x time heatmap of a per-core metric. + + Y axis is the NUMA domains, X axis is time (as in the line graphs of the + same directory) and the color is the domain's average metric value at each + monitoring step. It is the same data as the per-domain line graph, shown as + a heatmap; it lives next to it (all_numa/pinned_numa). + """ + components = numa_aggregated_components(bench, component_type, numa_nodes, pinned_cores) + if not components: + return 0 + unit = bench.get_metric_unit(component_type) + interval = bench.get_time_interval() + matrix = np.array([component.get_mean() for component in components], dtype=float) + domains = len(components) + samples = matrix.shape[1] + + trace = bench.get_trace() + title = f'{item_title} during "{bench.get_bench_name()}" benchmark job\n{args.title}\n\n Stressor: ' + title += f"{bench.workers()} x {bench.get_title_engine_name()} for {bench.duration()} seconds" + title += f"\n{bench.get_system_title()}" + graph_dir = output_dir.joinpath(f"{trace.get_name()}/{bench.get_bench_name()}/{component_type!s}") + if dir_suffix: + graph_dir = graph_dir.joinpath(dir_suffix) + graph = Graph( + args, + title, + "Time [seconds]", + "NUMA domain", + graph_dir, + f"{item_title} - heatmap", + show_source_file=trace, + title_note=title_note, + ) + ax = graph.get_ax() + image = ax.imshow( + matrix, + aspect="auto", + cmap="viridis", + interpolation="nearest", + extent=(0, samples * interval, domains - 0.5, -0.5), + ) + graph.fig.colorbar(image, ax=ax, fraction=0.046, pad=0.04, label=unit) + ax.set_yticks(range(domains)) + ax.set_yticklabels([component.get_full_name() for component in components]) + + # Separate box on the left listing each domain's cores in condensed form, + # like the component legend of the line graphs. + pinned_cpus = None + if pinned_cores is not None: + pinned_cpus = {int(name.split("_")[1]) for name in pinned_cores} + node_cores = [] + for component in components: + node = int(component.get_full_name().split()[1]) + cpus = numa_nodes[node] + if pinned_cpus is not None: + cpus = [cpu for cpu in cpus if cpu in pinned_cpus] + node_cores.append((node, cpus)) + legend = numa_cores_legend(ax, node_cores) + graph.needs_legend = False + graph.render(extra_legend=legend) + return 1 + + def generic_graph( args, output_dir, @@ -320,11 +545,13 @@ def generic_graph( names=None, dir_suffix=None, title_note=None, + components=None, ) -> int: outfile = f"{item_title}" trace = bench.get_trace() - components = bench.get_all_metrics(component_type, filter, names) + if components is None: + components = bench.get_all_metrics(component_type, filter, names) if not len(components): title = f"{item_title}: no {component_type!s} metric found" if filter: diff --git a/graph/hwgraph.py b/graph/hwgraph.py index 345bd77..f6678e5 100755 --- a/graph/hwgraph.py +++ b/graph/hwgraph.py @@ -19,7 +19,14 @@ try: from graph.chassis import graph_chassis from graph.common import fatal - from graph.graph import generic_graph, init_matplotlib, yerr_graph + from graph.graph import ( + generic_graph, + init_matplotlib, + numa_aggregated_components, + numa_distance_heatmap, + numa_performance_heatmap, + yerr_graph, + ) from graph.group import graph_group_env from graph.scaling import smp_scaling_graph from graph.trace import Event, Trace @@ -66,11 +73,18 @@ def _task_env_bench(trace_idx, bench_name, output_dir_str): count += graph_monitoring_metrics(args, trace, bench_name, output_dir) count += graph_fans(args, trace, bench_name, output_dir) count += graph_cpu(args, trace, bench_name, output_dir) + count += graph_cpu_numa(args, trace, bench_name, output_dir) count += graph_pdu(args, trace, bench_name, output_dir) count += graph_thermal(args, trace, bench_name, output_dir) return count +def _task_numa_distance(trace_idx, output_dir_str): + """Generate the per-host NUMA distance heatmap (once per trace).""" + global _pool_args + return numa_distance_heatmap(_pool_args, pathlib.Path(output_dir_str), _pool_args.traces[trace_idx]) + + def _task_chassis(bench_name, output_dir_str): """Generate chassis graphs for a single bench.""" global _pool_args @@ -185,6 +199,8 @@ def valid_traces(args): print(f"environment: rendering {len(benches)} jobs from {trace.get_filename()} ({trace.get_name()})") for bench_name in sorted(benches): tasks.append((_task_env_bench, trace_idx, bench_name, str(host_output_dir))) + # One NUMA distance heatmap per host, not tied to any benchmark. + tasks.append((_task_numa_distance, trace_idx, str(host_output_dir))) return tasks @@ -499,6 +515,65 @@ def graph_cpu(args, trace: Trace, bench_name: str, output_dir) -> int: return rendered_graphs +def graph_cpu_numa(args, trace: Trace, bench_name: str, output_dir) -> int: + """Render per-core CPU metrics aggregated by NUMA domain (one line per domain). + + Requires the NUMA topology in the trace; older traces without it are skipped. + """ + numa_nodes = trace.get_numa_nodes() + if not numa_nodes: + print(f"{bench_name}: no NUMA metric present in trace file, skipping.") + return 0 + rendered_graphs = 0 + bench = trace.bench(bench_name) + numa_graphs = { + "Core frequency per NUMA domain": MonitoringContextKeys.Freq, + "Core IPC per NUMA domain": MonitoringContextKeys.IPC, + "CPU Core power consumption per NUMA domain": MonitoringContextKeys.PowerConsumption, + } + # Metrics that also get a per-domain x time heatmap next to their line graph. + heatmap_metrics = {MonitoringContextKeys.Freq, MonitoringContextKeys.IPC} + # Like the per-core graphs: one rendering averaging every core of each domain + # (all_numa), and one restricted to the cores pinned during this job, grouped + # by the domains those pinned cores belong to (pinned_numa). + pinned_core_names = bench.pinned_core_names() + # Each rendering is a dir suffix, the pinned-core filter, and a title note. + renderings = [("all_numa", None, None)] # type: list + if pinned_core_names: + pinned_note = f"View limited to the pinned logical cores {bench.pinned_cpu_range()}" + renderings.append(("pinned_numa", pinned_core_names, pinned_note)) + + for graph_name, metric in numa_graphs.items(): + for dir_suffix, pinned_cores, title_note in renderings: + components = numa_aggregated_components(bench, metric, numa_nodes, pinned_cores) + if not components: + continue + rendered_graphs += generic_graph( + args, + output_dir, + bench, + metric, + graph_name, + dir_suffix=dir_suffix, + components=components, + title_note=title_note, + ) + # Companion heatmap: NUMA domain x time, color = per-domain value. + if metric in heatmap_metrics: + rendered_graphs += numa_performance_heatmap( + args, + output_dir, + bench, + metric, + graph_name, + numa_nodes, + dir_suffix=dir_suffix, + pinned_cores=pinned_cores, + title_note=title_note, + ) + return rendered_graphs + + def graph_pdu(args, trace: Trace, bench_name: str, output_dir) -> int: rendered_graphs = 0 bench = trace.bench(bench_name) diff --git a/graph/trace.py b/graph/trace.py index 38a30dc..063fa59 100644 --- a/graph/trace.py +++ b/graph/trace.py @@ -580,6 +580,27 @@ def get_sanitized_cpu_model(self): def get_sockets_count(self): return self.get_cpu()["sockets"] + def get_numa_nodes(self) -> dict[int, list[int]]: + """Return the {numa domain: [logical cores]} mapping. + + Returns an empty dict for older traces that did not record it. + """ + numa_nodes = self.get_cpu().get("numa_nodes") + if not numa_nodes: + return {} + # JSON object keys are strings; expose them as ints. + return {int(node): cores for node, cores in numa_nodes.items()} + + def get_numa_distances(self) -> dict[int, list[int]]: + """Return the NUMA distance matrix ({source node: [latency to each node]}). + + Returns an empty dict for older traces that did not record it. + """ + distances = self.get_cpu().get("numa_distances") + if not distances: + return {} + return {int(node): latencies for node, latencies in distances.items()} + def get_physical_cores(self): return self.get_cpu()["physical_cores"] diff --git a/hwbench/bench/test_numa.py b/hwbench/bench/test_numa.py index da49f55..d324122 100644 --- a/hwbench/bench/test_numa.py +++ b/hwbench/bench/test_numa.py @@ -57,6 +57,23 @@ def test_numa_simple(self): for index, numa_node in enumerate(cumulative_numa_nodes): assert self.get_bench_parameters(9 + index).get_pinned_cpu() == numa_node + def test_numa_dump(self): + """The CPU dump exposes the NUMA topology: node->cores and distance matrix.""" + dump = self.hw.get_cpu().dump() + # NUMA node -> logical cores mapping + assert dump["numa_domains"] == 8 + assert set(dump["numa_nodes"].keys()) == set(range(8)) + assert dump["numa_nodes"][0] == self.NUMA0 + assert dump["numa_nodes"][1] == self.NUMA1 + assert dump["numa_nodes"][7] == self.NUMA7 + # NUMA distance matrix: 8x8, local node is 10, node 0<->1 is 11, rest 12 + distances = dump["numa_distances"] + assert len(distances) == 8 + assert distances[0] == [10, 11, 12, 12, 12, 12, 12, 12] + for domain in range(8): + assert len(distances[domain]) == 8 + assert distances[domain][domain] == 10 + def test_numa(self): """Check numa syntax""" assert self.hw.logical_core_count() == 128 diff --git a/hwbench/engines/stressng_stream.py b/hwbench/engines/stressng_stream.py index e211fbe..93bdbb0 100644 --- a/hwbench/engines/stressng_stream.py +++ b/hwbench/engines/stressng_stream.py @@ -23,7 +23,7 @@ def run_cmd(self) -> list[str]: str(self.parameters.get_runtime()), "--metrics", "--yaml", - f"{self.name}.yaml", + f"{self.output_basename}.yaml", "--stream", str(self.parameters.get_engine_instances_count()), ] diff --git a/hwbench/environment/cpu.py b/hwbench/environment/cpu.py index 18d43ee..797f8c6 100644 --- a/hwbench/environment/cpu.py +++ b/hwbench/environment/cpu.py @@ -74,6 +74,10 @@ def get_logical_cores_in_numa_domain(self, numa_domain) -> list[int]: """Return logical cores in a numa domain.""" return self.numa.get_cores(numa_domain) + def get_numa_distances(self) -> dict[int, list[int]]: + """Return the NUMA distance matrix ({source node: [latency to each node]}).""" + return self.numa.get_distances() + def get_quadrants_count(self) -> int: """Return the number of quadrants.""" return self.numa.quadrants_count() @@ -93,6 +97,13 @@ def dump(self): "logical_cores": self.get_logical_cores_count(), "physical_cores": self.get_physical_cores_count(), "numa_domains": self.get_numa_domains_count(), + # Mapping of each NUMA domain to its logical cores, so consumers + # (e.g. hwgraph) can aggregate per-core metrics by NUMA domain. + "numa_nodes": { + node: self.get_logical_cores_in_numa_domain(node) for node in range(self.get_numa_domains_count()) + }, + # NUMA distance matrix between domains, for topology-aware features. + "numa_distances": self.get_numa_distances(), "sockets": self.get_sockets_count(), } diff --git a/hwbench/environment/numa.py b/hwbench/environment/numa.py index d3ee3d5..957a49d 100644 --- a/hwbench/environment/numa.py +++ b/hwbench/environment/numa.py @@ -40,6 +40,9 @@ def parse_cmd(self, stdout: bytes, _stderr: bytes): numa_distance = re.findall(r"(\d+): (.*)", line) if numa_distance: for source, latencies in numa_distance: + # Keep the full distance matrix (source node -> latency to each + # node); useful to reason about node locality/topology. + self.distances[int(source)] = [int(latency) for latency in latencies.split()] quadrant = self.__is_numa_node_in_quadrant(source) numa_dest = -1 for latency in latencies.split(): @@ -65,6 +68,7 @@ def __init__(self, out_dir: pathlib.Path): super().__init__(out_dir) self.numa_domains: dict[int, list[int]] = {} self.quadrants: list[list[int]] = [] + self.distances: dict[int, list[int]] = {} def count(self) -> int: return len(self.numa_domains) @@ -72,6 +76,10 @@ def count(self) -> int: def get_cores(self, numa_domain) -> list[int]: return self.numa_domains.get(numa_domain, []) + def get_distances(self) -> dict[int, list[int]]: + """Return the NUMA distance matrix: {source node: [latency to each node]}.""" + return self.distances + def quadrants_count(self) -> int: return len(self.quadrants) diff --git a/hwbench/environment/test_parse.py b/hwbench/environment/test_parse.py index 4973115..9450ffb 100644 --- a/hwbench/environment/test_parse.py +++ b/hwbench/environment/test_parse.py @@ -141,6 +141,13 @@ def test_parsing_numa_8_domains_with_llc(self): assert test_target.count() == 8 for domain in range(0, test_target.count()): assert len(test_target.get_cores(domain)) == 16 + # Full 8x8 distance matrix: local node is 10, node 0<->1 is 11, rest 12 + distances = test_target.get_distances() + assert len(distances) == 8 + assert distances[0] == [10, 11, 12, 12, 12, 12, 12, 12] + for domain in range(0, test_target.count()): + assert len(distances[domain]) == 8 + assert distances[domain][domain] == 10 def test_parsing_numa_20_domains_with_llc(self): d = pathlib.Path("./hwbench/tests/parsing/numa/20domainsllc") @@ -160,6 +167,12 @@ def test_parsing_numa_20_domains_with_llc(self): assert test_target.get_numa_nodes_in_quadrant(1) == [5, 6, 7, 8, 9] assert test_target.get_numa_nodes_in_quadrant(2) == [10, 11, 12, 13, 14] assert test_target.get_numa_nodes_in_quadrant(3) == [15, 16, 17, 18, 19] + # Full 20x20 distance matrix with a local distance of 10 on the diagonal + distances = test_target.get_distances() + assert len(distances) == 20 + for domain in range(0, test_target.count()): + assert len(distances[domain]) == 20 + assert distances[domain][domain] == 10 class TestParseIpmitool: