From 096329e625f51493f139f36a4a3c2b02b0887322 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Tue, 2 Jun 2026 13:22:03 +0000 Subject: [PATCH 1/3] Multi-objective: default 1D plots to the model-optimal point (Brownie Bee issue #6) For multi-objective runs the per-objective 1D plots were each drawn at that objective's own expected minimum when no selectedPoint was supplied, so the quality and cost rows described different factor settings. Default the selected_point to the Pareto front's compromise point (front_x_data[best_idx]) when the caller sends none, so both objectives' 1D plots are evaluated at the same point. emit_pareto_data now returns that optimal point for the caller to reuse. An explicit selectedPoint (a clicked point) still takes precedence. --- optimizerapi/optimizer.py | 16 +++++++++++++--- optimizerapi/plot_emitters.py | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/optimizerapi/optimizer.py b/optimizerapi/optimizer.py index 0c0cf17..088054b 100644 --- a/optimizerapi/optimizer.py +++ b/optimizerapi/optimizer.py @@ -346,21 +346,31 @@ def process_result( if optimizer.n_objectives == 1: _set_expected_minimum(result_details, result[0], optimizer.space) + optimal_point = None if optimizer.n_objectives == 2 and "pareto" in parsed.graphs_to_return: - emit_pareto_data(plots, optimizer) + optimal_point = emit_pareto_data(plots, optimizer) if optimizer.n_objectives == 2 and "single" in parsed.graphs_to_return: + # Default the 1D plots to the model's optimal (compromise) point + # so both objectives describe the *same* settings. Without an + # explicit point each objective falls back to its own expected + # minimum, leaving the two rows misaligned. + point = ( + parsed.selected_point + if parsed.selected_point is not None + else optimal_point + ) emit_json_single_plots( plots, result=result[0], prefix="objective_1", - selected_point=parsed.selected_point, + selected_point=point, ) emit_json_single_plots( plots, result=result[1], prefix="objective_2", - selected_point=parsed.selected_point, + selected_point=point, ) if parsed.include_model: diff --git a/optimizerapi/plot_emitters.py b/optimizerapi/plot_emitters.py index fb15408..1d70d5d 100644 --- a/optimizerapi/plot_emitters.py +++ b/optimizerapi/plot_emitters.py @@ -131,8 +131,15 @@ def emit_png_plots( _emit_current_figure(plots, f"single_{idx}") -def emit_pareto_data(plots: "list[Plot]", optimizer: object) -> None: - """Append the pareto-front payload (multi-objective only).""" +def emit_pareto_data(plots: "list[Plot]", optimizer: object) -> "list | None": + """Append the pareto-front payload (multi-objective only). + + Returns the x-space coordinates of the model's compromise/optimal point + (``front_x_data[best_idx]``) so the caller can use it as the default + ``selected_point`` for the per-objective 1D plots — without it, each + objective's plots default to its own expected minimum and the two rows end + up describing different settings. Returns ``None`` if no front is available. + """ front_x_data, front_y_data, obj1_error, obj2_error = get_Brownie_Bee_Pareto( optimizer, n_points=200 ) @@ -146,6 +153,10 @@ def emit_pareto_data(plots: "list[Plot]", optimizer: object) -> None: } plots.append({"id": "pareto_data", "plot": json_tricks.dumps(pareto_data)}) + if best_idx is None or len(front_x_data) == 0: + return None + return front_x_data[best_idx].tolist() + def _figure_to_b64(figure) -> str: buf = io.BytesIO() From 3003c42b47e643ed20499ccc4a194ae860efaf1c Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Tue, 2 Jun 2026 21:04:15 +0200 Subject: [PATCH 2/3] Multi-objective: fix frozen score histogram on Pareto point selection (Brownie Bee issue #10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_Brownie_Bee_1d_plot computes the histogram mean/std via model.predict() on the raw selected point, but the GP is fitted in transformed space. For all-numeric spaces the raw point lands outside the normalized domain, so the prediction collapses to the prior mean — the histogram stayed constant across every Pareto point (while the per-factor plots, which transform x_eval via dependence(), moved). The _get_brownie_bee_1d_plot_safe workaround only transformed for categorical points, leaving numeric spaces (e.g. cfps-multi) broken. Recompute the histogram in emit_json_single_plots via model.predict(space.transform([selected_point])); the default (selected_point is None) path still reuses the expected_minimum-derived value, which was already correct. Adds a numeric-only regression test. --- optimizerapi/plot_emitters.py | 38 ++++++++--- tests/test_e2e_pareto.py | 118 ++++++++++++++++++++++++++++++++++ tests/test_plot_emitters.py | 30 ++++++++- 3 files changed, 175 insertions(+), 11 deletions(-) diff --git a/optimizerapi/plot_emitters.py b/optimizerapi/plot_emitters.py index 1d70d5d..a176cd3 100644 --- a/optimizerapi/plot_emitters.py +++ b/optimizerapi/plot_emitters.py @@ -78,26 +78,46 @@ def emit_json_single_plots( X-space coordinates to highlight, or ``None`` to use the default. """ one_d_data = _get_brownie_bee_1d_plot_safe(result, x_eval=selected_point) - histogram_entry = one_d_data[-1] for i, dim_data in enumerate(one_d_data[:-1]): plots.append( {"id": f"{prefix}_{i}", "plot": json_tricks.dumps({"data": dim_data})} ) + mean, std = _histogram_mean_std(result, selected_point, one_d_data[-1]) plots.append( { "id": f"{prefix}_{len(one_d_data) - 1}", - "plot": json_tricks.dumps( - { - "histogram": { - "mean": float(numpy.ravel(histogram_entry[0])[0]), - "std": float(numpy.ravel(histogram_entry[1])[0]), - } - } - ), + "plot": json_tricks.dumps({"histogram": {"mean": mean, "std": std}}), } ) +def _histogram_mean_std( + result: Any, + selected_point: "list[str | float] | None", + fallback_entry: Any, +) -> "tuple[float, float]": + """Predicted-score mean/std for the histogram at ``selected_point``. + + When no point is given, ``get_Brownie_Bee_1d_plot`` derives these from + ``expected_minimum`` (correct), so we reuse its last entry. For an explicit + point it instead predicts on the *raw* point — but the model is fitted in + transformed space, so the raw point lands outside the normalized domain and + the prediction collapses to the prior mean (constant across Pareto points) + for numeric spaces, and errors for categorical ones. Predict on the + transformed point here so the histogram actually tracks the selection. + """ + if selected_point is None: + return ( + float(numpy.ravel(fallback_entry[0])[0]), + float(numpy.ravel(fallback_entry[1])[0]), + ) + model = result.models[-1] + mean, std = model.predict( + numpy.asarray(result.space.transform([selected_point])), return_std=True + ) + return float(numpy.ravel(mean)[0]), float(numpy.ravel(std)[0]) + + def emit_png_plots( plots: "list[Plot]", *, diff --git a/tests/test_e2e_pareto.py b/tests/test_e2e_pareto.py index 6ed673f..71bed27 100644 --- a/tests/test_e2e_pareto.py +++ b/tests/test_e2e_pareto.py @@ -33,6 +33,39 @@ "constraints": [], } +# Numeric-only multi-objective data (the built-in "CFPS multi objective" +# example). Unlike MULTI_OBJECTIVE_DATA above it has no categorical factor, so +# it exercises the all-numeric code path of get_Brownie_Bee_1d_plot — where the +# score histogram (predicted-score distribution at the selected point) used to +# be computed on the *raw* (untransformed) point and therefore never changed +# between Pareto points. +NUMERIC_MULTI_OBJECTIVE_DATA = [ + {"xi": [1, 40, 0.5], "yi": [0.608, 0.0833]}, + {"xi": [5, 120, 2.5], "yi": [1.9085, 2.0833]}, + {"xi": [3, 80, 1.5], "yi": [1.3638, 0.75]}, + {"xi": [5.5, 30, 0.25], "yi": [1.1523, 0.8542]}, + {"xi": [2, 140, 2], "yi": [1.0272, 1.5556]}, + {"xi": [0, 100, 1], "yi": [0.3739, 0.5556]}, + {"xi": [4, 60, 2.75], "yi": [1.2057, 1.3958]}, + {"xi": [3.5, 130, 0.75], "yi": [1.6283, 1.2431]}, + {"xi": [1.5, 70, 2.25], "yi": [0.8189, 0.7986]}, + {"xi": [0.5, 90, 0], "yi": [0.4268, 0.3472]}, +] + +NUMERIC_MULTI_OBJECTIVE_CONFIG = { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 4, + "kappa": 1.96, + "xi": 0.01, + "space": [ + {"type": "continuous", "name": "Magnesium", "from": 0, "to": 6}, + {"type": "discrete", "name": "Potassium", "from": 20, "to": 140}, + {"type": "continuous", "name": "DTT", "from": 0, "to": 3}, + ], + "constraints": [], +} + @pytest.mark.filterwarnings("ignore::DeprecationWarning") class TestE2EParetoFront: @@ -182,6 +215,91 @@ def test_selectedPoint_highlights_correct_point(self, app_client, api_key): "selectedPoint should be reflected in the single plot highlight" ) + def test_score_histogram_tracks_selected_point_numeric_space( + self, app_client, api_key + ): + """The score histogram must follow the selected Pareto point. + + Regression for the "histograms never change when clicking the Pareto + front" bug: get_Brownie_Bee_1d_plot computed the histogram mean/std via + model.predict() on the *raw* point, but the GP is fitted in transformed + space. For an all-numeric space the raw point sits far outside the + normalized domain, so every point collapsed to the prior mean — the + histogram was identical regardless of selection (while the per-factor + plots, which transform x_eval themselves, did move). + """ + + def histogram_for(selected_point): + payload = self.build_request( + data=NUMERIC_MULTI_OBJECTIVE_DATA, + optimizer_config=NUMERIC_MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["single"], + "graphFormat": "json", + "selectedPoint": selected_point, + }, + ) + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + assert response.status_code == 200 + plots = response.get_json()["plots"] + # The histogram is the last entry per objective: one plot per space + # dimension (3) followed by the histogram at index 3. + out = {} + for obj in ("objective_1", "objective_2"): + hist_plot = next(p for p in plots if p["id"] == f"{obj}_3") + out[obj] = json.loads(hist_plot["plot"])["histogram"] + return out + + # First fetch the Pareto front and pick two genuinely different + # trade-offs: best quality vs best cost. + front_payload = self.build_request( + data=NUMERIC_MULTI_OBJECTIVE_DATA, + optimizer_config=NUMERIC_MULTI_OBJECTIVE_CONFIG, + extras={"graphs": ["pareto"], "graphFormat": "json"}, + ) + front_resp = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=front_payload, + content_type="application/json", + ) + assert front_resp.status_code == 200 + pareto = json.loads( + next(p for p in front_resp.get_json()["plots"] if p["id"] == "pareto_data")[ + "plot" + ] + ) + front_x = pareto["front_x_data"] + front_y = pareto["front_y_data"] + i_best_quality = min(range(len(front_y)), key=lambda i: front_y[i][0]) + i_best_cost = min(range(len(front_y)), key=lambda i: front_y[i][1]) + + # Sanity: the front must actually offer a trade-off, else the test is + # vacuous. + assert front_y[i_best_quality][0] != front_y[i_best_cost][0] + assert front_y[i_best_quality][1] != front_y[i_best_cost][1] + + at_quality = histogram_for(front_x[i_best_quality]) + at_cost = histogram_for(front_x[i_best_cost]) + + for obj in ("objective_1", "objective_2"): + assert at_quality[obj]["mean"] != at_cost[obj]["mean"], ( + f"{obj} histogram mean did not change between two different " + f"Pareto points (frozen histogram bug)" + ) + + # And the histogram mean should track the predicted score at the + # selected point (front_y), not collapse to a constant prior mean. + assert at_quality["objective_1"]["mean"] == pytest.approx( + front_y[i_best_quality][0], abs=0.1 + ) + assert at_cost["objective_2"]["mean"] == pytest.approx( + front_y[i_best_cost][1], abs=0.1 + ) + def test_full_pareto_exploration_workflow(self, app_client, api_key): """ Test the complete pareto exploration workflow: diff --git a/tests/test_plot_emitters.py b/tests/test_plot_emitters.py index f242509..e34dc51 100644 --- a/tests/test_plot_emitters.py +++ b/tests/test_plot_emitters.py @@ -50,6 +50,26 @@ def test_emit_json_single_plots_supports_different_prefixes(fake_brownie_1d): assert ids[-1] == "objective_2_3" +class _FakeModel: + def predict(self, X, return_std=False): + # Make the histogram depend on the transformed point so the test would + # catch a regression that ignored it. + value = float(numpy.ravel(X)[0]) + return numpy.array([value]), numpy.array([0.5]) + + +class _FakeResult: + """Minimal stand-in for an OptimizeResult with a transform + model.""" + + def __init__(self): + self.models = [_FakeModel()] + self.space = self + + def transform(self, points): + # Identity transform; just records that it was called with the point. + return numpy.asarray(points, dtype=float) + + def test_emit_json_single_plots_passes_selected_point_through(monkeypatch): captured = {} @@ -62,6 +82,12 @@ def _stub(result, x_eval=None): ) plots: list = [] - sp = [50, 833, "Whipped cream"] - emit_json_single_plots(plots, result=object(), prefix="single_0", selected_point=sp) + sp = [50, 833] + emit_json_single_plots( + plots, result=_FakeResult(), prefix="single_0", selected_point=sp + ) assert captured["x_eval"] == sp + # With a selected point the histogram is predicted at the transformed point, + # not taken from get_Brownie_Bee_1d_plot's (buggy) last entry. + histogram_payload = json.loads(plots[-1]["plot"]) + assert histogram_payload["histogram"]["mean"] == 50.0 From 5fd94477b72cfce64a390f412ebd8360431e4d0c Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Tue, 2 Jun 2026 22:02:53 +0200 Subject: [PATCH 3/3] Fix mypy no-any-return in emit_pareto_data front_x_data[best_idx].tolist() returns Any (numpy is untyped), but the function is declared -> list | None. Cast the return so `mypy optimizerapi` (the CI type-check step) passes. Pre-existing since d204b2d; surfaced now because it blocks the build. --- optimizerapi/plot_emitters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optimizerapi/plot_emitters.py b/optimizerapi/plot_emitters.py index a176cd3..c77f6a6 100644 --- a/optimizerapi/plot_emitters.py +++ b/optimizerapi/plot_emitters.py @@ -8,7 +8,7 @@ import base64 import io -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import json_tricks import matplotlib.pyplot as plt @@ -175,7 +175,7 @@ def emit_pareto_data(plots: "list[Plot]", optimizer: object) -> "list | None": if best_idx is None or len(front_x_data) == 0: return None - return front_x_data[best_idx].tolist() + return cast("list", front_x_data[best_idx].tolist()) def _figure_to_b64(figure) -> str: