Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 7 additions & 11 deletions demo/serve_nlp.py
Original file line number Diff line number Diff line change
Expand Up @@ -1019,18 +1019,14 @@ def main() -> None:
reference_texts, reference_labels = reference_corpus if reference_corpus is not None else (None, None)
xpl.fit(reference_texts, y=reference_labels, cache_dir=similar_cache_dir)

if config.recompute:
# Drop this text set's cached contributions so the steps below recompute + overwrite. The
# explainer owns the key (it covers the model and backend), so it also owns the deletion —
# only *this* model+backend's entry goes, other backends' caches are left intact. The
# embeddings are dropped by compute_embeddings(recompute=True) below, the projection is re-fitted.
logger.info("--recompute: dropping cached %s contributions + projection for these texts", config.attribution)
xpl.clear_cache(sentences, explain_dir)

# Both steps load from cache if one exists for these exact texts, otherwise they compute and write
# it. So the first run is slow; later runs only pay the model load.
# it. So the first run is slow; later runs only pay the model load. --recompute skips the lookup
# and overwrites only *this* model+backend's entry (other backends' caches are left intact); the
# embeddings are refreshed by compute_embeddings(recompute=True) below, the projection is re-fitted.
explain_cache = xpl.cache_path(sentences, explain_dir)
if explain_cache.exists():
if config.recompute:
logger.info("--recompute: recomputing %s contributions + projection for these texts", config.attribution)
elif explain_cache.exists():
logger.info("Contributions cache hit (%s) — loading %s", config.attribution, explain_cache)
else:
logger.info(
Expand All @@ -1042,7 +1038,7 @@ def main() -> None:
# immutable and holds no model, no backend and no explainer handle; ground truth rides on it
# (never on xpl), which is why a cache hit still gets *this* run's y_true attached.
dataset_label = str(config.dataset_path) if config.is_local_dataset else config.dataset_name
explanation = xpl.explain(sentences, y=y_true, cache_dir=explain_dir)
explanation = xpl.explain(sentences, y=y_true, cache_dir=explain_dir, recompute=config.recompute)

# Step 4 — everything downstream reads the artifact as an argument.
# Saved per space, so switching --embedding-space never reloads a layout drawn in another one, and
Expand Down
37 changes: 32 additions & 5 deletions shapash/explainer/nlp_explainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from __future__ import annotations

import logging
import re
import threading
import zipfile
from dataclasses import replace
Expand Down Expand Up @@ -58,6 +59,23 @@ def _cache_file(data_hash: str, cache_dir: Path) -> Path:
return cache_dir / f"{data_hash}.xpl"


_RELEASE_RE = re.compile(r"\d+\.\d+\.\d+")


def _cache_release() -> str:
"""The shapash release (``major.minor.patch``) that scopes explanation cache entries.

Every release, patch releases included, gets its own entries: a patch may fix how a backend
computes, and a stale entry looks exactly like a correct one. Anything after the patch number is
dropped, so a release candidate shares the final release's entries, and a development build
(``2.10.1.dev13+g<hash>.d<date>``) keeps one key across commits and days instead of missing on
each. A source tree that was never installed has no parsable version and falls back to the raw
string.
"""
match = _RELEASE_RE.match(__version__)
return match.group() if match else __version__


def _load_cached(cache_path: Path | None) -> NlpExplanation | None:
"""Read a cached explanation, or return ``None`` when the entry cannot be read.

Expand Down Expand Up @@ -311,6 +329,7 @@ def explain(
X: list[str] | pd.Series,
y: list | pd.Series | None = None,
cache_dir: str | Path | None = None,
recompute: bool = False,
) -> NlpExplanation:
"""Compute token-level contributions and model predictions, and return them as an artifact.

Expand Down Expand Up @@ -348,6 +367,13 @@ def explain(
default. One directory can be shared across models and backends: the hash identifies
them, so entries cannot collide. Needs a model with an identity — a ``TextModel`` or a
pipeline; a bare scoring callable raises ``ValueError``.
recompute : bool, optional
Skip both the in-memory and the disk cache and run the backend again, then store the
fresh result in their place. The previous entry is only overwritten once the new one is
computed, so a failed recompute leaves it in place. Use it when a cached result may be
stale, for example after changing backend code on a development build, whose version
does not change between commits, or to draw again from a sampling backend such as LIME.
Default ``False``.

Returns
-------
Expand All @@ -370,11 +396,11 @@ def explain(
# ``_COMPUTED_FIELDS`` half is meaningful here — its own texts/index/y_true belong to
# whoever computed it first. See ``relabelled`` at the return below.
cached = getattr(self, "_computed_cache", None)
if cached is not None and cached[0] == new_hash:
if not recompute and cached is not None and cached[0] == new_hash:
computed = cached[1]
else:
cache_path = _cache_file(new_hash, Path(cache_dir)) if cache_dir is not None else None
computed = _load_cached(cache_path)
computed = None if recompute else _load_cached(cache_path)

if computed is None:
contributions = self.backend.run_explainer(text_list)
Expand Down Expand Up @@ -934,8 +960,9 @@ def _compute_key(self, text_list: list[str]) -> str:
weights), the backend's registered ``name``, its explainer settings, the explanation space,
baseline token and masking it reports (constructor choices, not class constants — a
probability-space and a logit-space ``nlp_shap`` share every other part of the key), and
``label_names`` (which fixes the column order of ``y_prob``). The shapash version is in it too: a
release may change how backends compute or cut units, and must not read an older one's entries.
``label_names`` (which fixes the column order of ``y_prob``). The shapash release
(``major.minor.patch``, see :func:`_cache_release`) is in it too: any release may change how
backends compute or cut units, and must not read an older one's entries.

Keying on the texts alone — as this once did — means swapping the backend (SHAP -> LIG),
the model, or the label order and pointing at the same ``cache_dir`` silently reloads the
Expand All @@ -958,7 +985,7 @@ def _compute_key(self, text_list: list[str]) -> str:
# selects the string path without touching any other part of the key.
masking = getattr(backend, "masking", None)
backend_id = f"{type(backend).name}:{args!r}:{compute_args!r}:{space}:{baseline!r}:{masking}"
return hash_corpus(text_list, f"{__version__}|{model_id}|{backend_id}|{self.label_names!r}")
return hash_corpus(text_list, f"{_cache_release()}|{model_id}|{backend_id}|{self.label_names!r}")

def _require_text_model(self) -> TextModel:
text_model = getattr(self, "_text_model", None)
Expand Down
65 changes: 63 additions & 2 deletions tests/unit_tests/explainer/test_nlp_explainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -546,6 +546,27 @@ def test_key_is_order_sensitive(self):
xpl = _explainer()
self.assertNotEqual(xpl._compute_key(["a", "b"]), xpl._compute_key(["b", "a"]))

def _key_under(self, version):
with patch("shapash.explainer.nlp_explainer.__version__", version):
return _explainer()._compute_key(_SAMPLE_TEXTS)

def test_every_release_changes_the_key(self):
# A patch may fix how a backend computes, so it must not read the previous release's entries.
self.assertNotEqual(self._key_under("2.10.0"), self._key_under("2.10.1"))
self.assertNotEqual(self._key_under("2.10.1"), self._key_under("2.11.0"))

def test_a_dev_build_keeps_one_key_across_commits_and_days(self):
# The full version string carries the commit hash and, on a dirty tree, a date.
clean = self._key_under("2.10.1.dev13+gabc1234")
self.assertEqual(clean, self._key_under("2.10.1.dev14+gdef5678.d20261103"))
self.assertEqual(clean, self._key_under("2.10.1"), "a dev build shares its target release's key")

def test_a_release_candidate_shares_the_final_release_key(self):
self.assertEqual(self._key_under("2.11.0rc1"), self._key_under("2.11.0"))

def test_an_uninstalled_source_tree_falls_back_to_the_raw_version(self):
self.assertNotEqual(self._key_under("0+unknown"), self._key_under("2.10.0"))


def _read_meta(path) -> dict:
with zipfile.ZipFile(path) as zf:
Expand Down Expand Up @@ -616,8 +637,8 @@ def test_cache_path_points_at_the_file_explain_writes(self):
self.assertTrue(path.exists(), "cache_path disagrees with where explain() wrote")

def test_stale_layout_is_a_miss_not_a_crash(self):
# The cache key covers texts/model/backend but not the shapash version, so a layout change
# leaves entries the new reader cannot parse. The entry must be recomputed and overwritten.
# The cache key covers texts/model/backend and the release, not the commit, so a layout change
# on a development build can leave entries the new reader cannot parse. The entry must be recomputed and overwritten.
with tempfile.TemporaryDirectory() as cache_dir:
_explainer(backend=_MarkerBackend(marker=4.0)).explain(_SAMPLE_TEXTS, cache_dir=cache_dir)
path = _explainer().cache_path(_SAMPLE_TEXTS, cache_dir)
Expand Down Expand Up @@ -662,6 +683,46 @@ def test_clear_cache_forces_a_recompute(self):
xpl.explain(_SAMPLE_TEXTS, cache_dir=cache_dir)
self.assertEqual(backend.calls, 2, "clear_cache must defeat the in-memory guard too")

def test_recompute_bypasses_the_in_memory_guard(self):
backend = _MarkerBackend()
xpl = _explainer(backend=backend)
xpl.explain(_SAMPLE_TEXTS)
xpl.explain(_SAMPLE_TEXTS, recompute=True)
self.assertEqual(backend.calls, 2)

def test_recompute_bypasses_the_disk_cache_and_overwrites_it(self):
with tempfile.TemporaryDirectory() as cache_dir:
_explainer(backend=_MarkerBackend(marker=1.0)).explain(_SAMPLE_TEXTS, cache_dir=cache_dir)

# Same key (marker is not part of it), different output: stands in for a stale entry.
backend = _MarkerBackend(marker=2.0)
explanation = _explainer(backend=backend).explain(_SAMPLE_TEXTS, cache_dir=cache_dir, recompute=True)
self.assertEqual(backend.calls, 1, "recompute must not read the disk entry")
np.testing.assert_allclose(explanation.values[0], 2.0)

reloaded_backend = _MarkerBackend(marker=3.0)
reloaded = _explainer(backend=reloaded_backend).explain(_SAMPLE_TEXTS, cache_dir=cache_dir)
self.assertEqual(reloaded_backend.calls, 0)
np.testing.assert_allclose(reloaded.values[0], 2.0, err_msg="the recomputed result was not written")

def test_a_failed_recompute_keeps_the_previous_entry(self):
with tempfile.TemporaryDirectory() as cache_dir:
backend = _MarkerBackend(marker=1.0)
xpl = _explainer(backend=backend)
xpl.explain(_SAMPLE_TEXTS, cache_dir=cache_dir)
path = xpl.cache_path(_SAMPLE_TEXTS, cache_dir)

with (
patch.object(backend, "run_explainer", side_effect=MemoryError("boom")),
self.assertRaises(MemoryError),
):
xpl.explain(_SAMPLE_TEXTS, cache_dir=cache_dir, recompute=True)

self.assertTrue(path.exists(), "a failed recompute dropped the cached entry")
np.testing.assert_allclose(NlpExplanation.load(path).values[0], 1.0)
np.testing.assert_allclose(xpl.explain(_SAMPLE_TEXTS).values[0], 1.0)
self.assertEqual(backend.calls, 1, "the in-memory entry survives a failed recompute too")

def test_clear_cache_leaves_other_backends_entries_intact(self):
with tempfile.TemporaryDirectory() as cache_dir:
keeper = _MarkerBackend(marker=1.0)
Expand Down
Loading