diff --git a/.github/ci-constraints.txt b/.github/ci-constraints.txt new file mode 100644 index 000000000..240dd8aa7 --- /dev/null +++ b/.github/ci-constraints.txt @@ -0,0 +1,2 @@ +# Official wheel bypasses intermittent HTTP 503 responses from the docdata index. +docdata @ https://files.pythonhosted.org/packages/20/d6/ffc73563941fea7bc2bc5c0afe425d5b24a2a4c2348693125cb061fe4425/docdata-0.0.5-py3-none-any.whl#sha256=7e54713b4fe8d87b6483cafa83a6f119344a4f3163799de08b8cb97436a215be diff --git a/.github/mypy-baseline.txt b/.github/mypy-baseline.txt index 15c1c7c63..7fdcfb8c0 100644 --- a/.github/mypy-baseline.txt +++ b/.github/mypy-baseline.txt @@ -1 +1 @@ -1172 +1131 diff --git a/.github/workflows/github-actions-python-package.yml b/.github/workflows/github-actions-python-package.yml index cbab436b8..b35db27f1 100644 --- a/.github/workflows/github-actions-python-package.yml +++ b/.github/workflows/github-actions-python-package.yml @@ -6,6 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ["3.11", "3.12", "3.13"] @@ -18,12 +19,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e .["dev"] --extra-index-url https://download.pytorch.org/whl/cpu - pip install mypy + python -m pip install -e ".[dev]" -c .github/ci-constraints.txt --extra-index-url https://download.pytorch.org/whl/cpu + python -m pip check - name: Lint with ruff run: | - ruff check dicee/ --line-length=200 + ruff check dicee/ benchmarks/ --line-length=200 - name: "Type check with mypy (ratchet: error count must not increase)" run: | @@ -34,8 +35,12 @@ jobs: # add fresh type errors while existing debt is paid down gradually. # If you fix errors, lower .github/mypy-baseline.txt to lock in the # improvement; only raise it if a change unavoidably adds errors. - mypy dicee/ --config-file=pyproject.toml > mypy_output.txt 2>&1 || true + MYPY_EXIT=0 + mypy dicee/ --config-file=pyproject.toml --python-version "${{ matrix.python-version }}" > mypy_output.txt 2>&1 || MYPY_EXIT=$? cat mypy_output.txt + if [ "$MYPY_EXIT" -gt 1 ]; then + exit "$MYPY_EXIT" + fi CURRENT_ERRORS=$(grep -oP 'Found \K[0-9]+(?= errors?)' mypy_output.txt || echo 0) BASELINE_ERRORS=$(cat .github/mypy-baseline.txt) echo "mypy errors: $CURRENT_ERRORS (baseline: $BASELINE_ERRORS)" diff --git a/.gitignore b/.gitignore index 2607f72c5..0e6b44156 100644 --- a/.gitignore +++ b/.gitignore @@ -181,8 +181,17 @@ cython_debug/ .cursorindexingignore KGs/ Experiments/ -# Benchmark reports are local artifacts, not repository content. +# Benchmark reports are local artifacts; recipes, adapters and published scores are tracked. /benchmarks/**/*.json +!/benchmarks/adapters/*.json +!/benchmarks/adapters/**/*.json +!/benchmarks/plus_h/*.json +!/benchmarks/ultraquery/*.json +!/benchmarks/cqa/tests/*.json +/results/ +.benchmark-downloads/ +.fetch-*/ +.models-*/ checkpoints/ *.zip Literal_KGs/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 75e553d0a..b4e38efce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,9 +17,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Sharded checkpoint save/resume for the `torchFSDP` trainer - Adaptive Momentum Weight Averaging and soft-label support - `--log_level` CLI flag; TriX model +- KG-ICL (`KGICL`): official checkpoints in pure PyTorch with seeded, batch-independent prompt graphs, deterministic fused Triton inference, parity fixtures against the pinned official code with five upstream defects corrected, zero-shot benchmarks, and pinned +H and UltraQuery adapter recipes +- Complex query answering (`dicee.query_answering`): CQD beam search over KGEs and KGFMs with learned score adapters, negated query types, native UltraQuery, GNN-QE, QTO, CQD, CQD-Hybrid, ConE and CLMPT checked against their upstream implementations, and resumable filtered evaluation with exact expected ranks under ties +- Reproducible UltraQuery and +H benchmarks (`python -m benchmarks.cqa`): frozen studies verified against the pinned upstream implementations, a pinned Docker runtime, paired adapter, answer-filter and inference-graph ablations ### Changed - KGFM inference accelerated (fused kernels, query reuse, Flock execution) +- Deterministic KGFM message passing splits hub nodes into balanced, order-preserving partial sums and avoids per-query host synchronization; predictions are bitwise unchanged +- `KGE.answer_multi_hop_query` uses the shared query engine and accepts score adapters - Regression bounds updated for the new embedding initialization (AConEx, CoKE) - DeCaL initialization delegates embedding setup to dedicated methods (part of #427) - CI ratchets the mypy error count instead of `continue-on-error` diff --git a/README.md b/README.md index 989699ed7..ac4bf5007 100644 --- a/README.md +++ b/README.md @@ -351,7 +351,7 @@ The run reuses configuration and serialized artifacts from the existing experime ```bash dicee --continual_learning "KeciFamilyRun" --path_single_kg "KGs/Family/family-benchmark_rich_background.owl" --model Keci --backend rdflib --eval_model None ``` -The continual directory should contain the stored configuration and serialized training data (for example `configuration.json`, `memory_map_train_set.npy`, and mapping files `entity_to_idx`/`relation_to_idx` in `.csv` or legacy `.p` format). +The continual directory should contain the stored configuration and serialized training data (for example `configuration.json`, `memory_map_train_set.npy`, and mapping files `entity_to_idx`/`relation_to_idx` in `.csv` or `.p` format). If `--eval_model` is set, evaluation runs after training using stored indexed artifacts. If `--eval_model None`, no evaluation is executed. Periodic evaluation and weight-averaging callbacks are also supported in continual training. @@ -491,54 +491,101 @@ predictions = model.predict_topk(h=["Mongolia"], t=["Asia"], topk=5) -## Multi-Hop Query Answering -
EPFO Queries (1p, 2p, 3p, 2i, 3i, ip, pi, 2u, up) +## Complex Query Answering (CQA) + +Answer path, intersection, union, and negated queries with [CQD](https://arxiv.org/abs/2011.03459) +over KGEs or [ULTRA](https://arxiv.org/abs/2310.04562)/[TRIX](https://arxiv.org/abs/2502.19512)/[Flock](https://arxiv.org/abs/2510.01510), +with optional learned score adapters. [ConE](https://arxiv.org/abs/2110.13715), +[CLMPT](https://arxiv.org/abs/2402.12954), [CQD-Hybrid](https://arxiv.org/abs/2410.12537), +[GNN-QE](https://proceedings.mlr.press/v162/zhu22c.html), [QTO](https://proceedings.mlr.press/v202/bai23b.html), +and [UltraQuery](https://arxiv.org/abs/2404.07198) are available as native DICE implementations through `dicee.query_answering.methods`. + +
+Examples: 1-hop, 2-hop, and 3-hop queries + +`KGE` loads both ordinary KGEs and saved KGFM experiments. Keep the KGFM's saved +graph and vocabulary files alongside its weights. Save an adapter fitted for the +chosen backbone and t-norm as `adapter.json` in that experiment folder. +Adapter training defaults to linear `context_scores` (16 parameters); the shipped +recipes use it with the product t-norm. ```python +import numpy as np + from dicee import KGE +from dicee.models.graph_model import GraphKGE +from dicee.query_answering import QueryContext, QueryScoreAdapter -# Load pre-trained model -model = KGE(path="...") +# Choose one saved DICE experiment for the family graph. +model = KGE(path="Experiments/complex-family") # KGE: ComplEx +# model = KGE(path="Experiments/ultra-family") # KGFM: ULTRA +# model = KGE(path="Experiments/trix-family") # KGFM: TRIX +# model = KGE(path="Experiments/flock-family") # KGFM: Flock + +adapter = QueryScoreAdapter.load(f"{model.path}/adapter.json", model=model.model) + +# KGFMs reuse their saved graph; context-aware KGE adapters need observed facts. +context = None +if not isinstance(model.model, GraphKGE): + context = QueryContext( + np.load(f"{model.path}/train_set.npy"), model.num_entities, model.num_relations, + ) # 1-hop: Who are the siblings of F9M167? -# Query: ?E : ∃E.hasSibling(E, F9M167) +# Query: hasSibling(F9M167, E) predictions = model.answer_multi_hop_query( query_type="1p", - query=('http://www.benchmark.org/family#F9M167', - ('http://www.benchmark.org/family#hasSibling',)), - tnorm="min", k=3 + query=("http://www.benchmark.org/family#F9M167", + ("http://www.benchmark.org/family#hasSibling",)), + adapter=adapter, context=context, tnorm="prod", k=3, ) -# => [('F9F141', 0.99), ('F9M157', 0.98), ...] # 2-hop: To whom is a sibling of F9M167 married? -# Query: ?D : ∃E.Married(D,E) ∧ hasSibling(E, F9M167) +# Query: ∃E. hasSibling(F9M167, E) ∧ married(E, D) predictions = model.answer_multi_hop_query( query_type="2p", query=("http://www.benchmark.org/family#F9M167", ("http://www.benchmark.org/family#hasSibling", "http://www.benchmark.org/family#married")), - tnorm="min", k=3 + adapter=adapter, context=context, tnorm="prod", beam_size=64, k=3, ) -# => [('F9F158', 0.95), ('F9M142', 0.93), ...] -# 3-hop: What type of people are married to a sibling of F9M167? -# Query: ?T : ∃D.type(D,T) ∧ Married(D,E) ∧ hasSibling(E, F9M167) +# 3-hop: What types do those spouses have? +# Query: ∃E,D. hasSibling(F9M167, E) ∧ married(E, D) ∧ type(D, T) predictions = model.answer_multi_hop_query( query_type="3p", query=("http://www.benchmark.org/family#F9M167", ("http://www.benchmark.org/family#hasSibling", "http://www.benchmark.org/family#married", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type")), - tnorm="min", k=5 + adapter=adapter, context=context, tnorm="prod", beam_size=64, k=5, ) -# => [('Person', 0.99), ('Male', 0.99), ('Father', 0.98), ...] +# Each call returns a list of (entity, score) pairs. ``` -**📖 [See multi-hop query examples →](tests/test_answer_multi_hop_query.py)** -**Supported query types:** `1p` (1-hop projection), `2p` (2-hop), `3p` (3-hop), `2i` (2-way intersection), `3i` (3-way intersection), `ip` (intersection-projection), `pi` (projection-intersection), `2u` (2-way union), `up` (union-projection) +`beam_size` limits intermediate candidates; `k` limits returned answers. +Set `adapter = None` to use unadapted sigmoid scores.
+See the [CQA guide](docs/guides/multi_hop_queries.md) for adapters and method evaluation. + +**Benchmarks.** [`benchmarks/cqa`](benchmarks/cqa/README.md) evaluates these methods on the +23 UltraQuery datasets and on **+H**: FB15k237+H, NELL995+H and ICEWS18+H from +[Gregucci et al.](https://arxiv.org/abs/2410.12537), whose hard answers are balanced across the +number of links that must be predicted to reach them. Select any methods, datasets and query +types; `--split` is required: + +```bash +python -m benchmarks.cqa plus_h evaluate --split valid --output results/plus-h-subset \ + --methods cqd cqd-hybrid qto --datasets FB15k237+H NELL995+H --query-types 2p 3p +``` + +The verified workflow freezes inputs and query batches, checks validation predictions against +the pinned upstream implementations, runs in a pinned Docker image, and reports sort-based and +expected random-tie metrics with paired adapter controls. The [protocol](benchmarks/cqa/PROTOCOL.md) +records every known deviation from the publications. + ## Literal Prediction
Predicting Numeric/Literal Values @@ -675,6 +722,30 @@ sampling budgets, reproducibility, training, and verification against upstream.
+[KG-ICL](docs/kgicl.md) answers queries in context, from prompt graphs around a few +example facts of the query relation. The official checkpoints run in pure PyTorch, +with fused Triton kernels for CUDA inference. Scores and gradients are verified +against the official implementation, with five upstream defects corrected. + +
+KG-ICL checkpoint download and inference + +```bash +mkdir -p checkpoints/kgicl/KG-ICL-6L +wget -P checkpoints/kgicl/KG-ICL-6L https://raw.githubusercontent.com/nju-websoft/KG-ICL/6a3166e347ae468acdfb30a70a2cf3608b66b8f1/checkpoint/KG-ICL-6L/model_best.tar + +python -m dicee --model KGICL --dataset_dir KGs/UMLS \ + --kgicl_checkpoint checkpoints/kgicl/KG-ICL-6L/model_best.tar --num_epochs 0 \ + --trainer torchCPUTrainer --scoring_technique NegSample \ + --batch_size 8 --eval_model test +``` + +Prompt examples are drawn with a seed per graph and relation, so predictions are +deterministic. The [KG-ICL guide](docs/kgicl.md) covers prompt graphs, the official +evaluation protocol, training, the corrected defects, and verification. + +
+ ## KGFM Link Prediction Test-set entity prediction with released checkpoints and no fine-tuning, using @@ -685,7 +756,8 @@ Tie strategy: **pessimistic** (worst rank among exactly equal scores after filte
Show results -**Bold** marks the best result per dataset and metric; **Yes** marks target graphs used in pretraining. +**Bold** marks the best result per dataset and metric; **Yes** marks target graphs used in pretraining; +**Partial** marks targets of which KG-ICL saw a subgraph in pretraining (FB V1 and NELL V1; [counts](docs/kgfm_benchmarks.md#kinship-and-nell-variants)). | Dataset | Model | Target graph used in pretraining? | MRR | Hits@1 | Hits@3 | Hits@10 | |---|---|:---:|---:|---:|---:|---:| @@ -694,64 +766,78 @@ Tie strategy: **pessimistic** (worst rank among exactly equal scores after filte | YAGO3-10 | ULTRA-50g | Yes | **0.5769** | **0.4966** | **0.6259** | **0.7203** | | YAGO3-10 | TRIX | No | 0.4094 | 0.3024 | 0.4574 | 0.6265 | | YAGO3-10 | Flock | No | 0.3998 | 0.3092 | 0.4526 | 0.5636 | +| YAGO3-10 | KG-ICL | No | 0.3792 | 0.2684 | 0.4356 | 0.5938 | | NELL-995-h25† | ULTRA-3g | No | 0.3557 | 0.2771 | 0.4001 | 0.5023 | | NELL-995-h25† | ULTRA-4g | Yes | **0.4032** | **0.3218** | **0.4476** | **0.5591** | | NELL-995-h25† | ULTRA-50g | Yes | 0.3991 | 0.3169 | 0.4448 | 0.5577 | | NELL-995-h25† | TRIX | No | 0.3729 | 0.2939 | 0.4136 | 0.5232 | | NELL-995-h25† | Flock | No | 0.3672 | 0.2868 | 0.4100 | 0.5199 | +| NELL-995-h25† | KG-ICL | Partial | 0.3375 | 0.2700 | 0.3716 | 0.4661 | | NELL-995-h50† | ULTRA-3g | No | 0.3480 | 0.2618 | 0.3960 | 0.5108 | | NELL-995-h50† | ULTRA-4g | Yes | **0.3975** | **0.3059** | **0.4468** | **0.5753** | | NELL-995-h50† | ULTRA-50g | Yes | 0.3828 | 0.2938 | 0.4375 | 0.5516 | | NELL-995-h50† | TRIX | No | 0.3637 | 0.2769 | 0.4057 | 0.5309 | | NELL-995-h50† | Flock | No | 0.3646 | 0.2730 | 0.4126 | 0.5423 | +| NELL-995-h50† | KG-ICL | Partial | 0.3374 | 0.2618 | 0.3760 | 0.4805 | | NELL-995-h75† | ULTRA-3g | No | 0.3298 | 0.2477 | 0.3741 | 0.4861 | | NELL-995-h75† | ULTRA-4g | Yes | **0.3732** | **0.2848** | 0.4164 | **0.5470** | | NELL-995-h75† | ULTRA-50g | Yes | 0.3640 | 0.2800 | **0.4180** | 0.5239 | | NELL-995-h75† | TRIX | No | 0.3428 | 0.2593 | 0.3798 | 0.5075 | | NELL-995-h75† | Flock | No | 0.3469 | 0.2585 | 0.3882 | 0.5227 | +| NELL-995-h75† | KG-ICL | Partial | 0.3286 | 0.2543 | 0.3644 | 0.4699 | | NELL-995-h100 | ULTRA-3g | No | 0.3159 | 0.2368 | 0.3546 | 0.4713 | | NELL-995-h100 | ULTRA-4g | Yes | 0.3560 | 0.2719 | 0.3940 | 0.5239 | | NELL-995-h100 | ULTRA-50g | Yes | **0.3619** | **0.2803** | **0.4076** | **0.5271** | | NELL-995-h100 | TRIX | No | 0.3354 | 0.2504 | 0.3720 | 0.5053 | | NELL-995-h100 | Flock | No | 0.3397 | 0.2481 | 0.3840 | 0.5202 | +| NELL-995-h100 | KG-ICL | Partial | 0.3308 | 0.2552 | 0.3675 | 0.4810 | | FB15k-237 | ULTRA-3g | Yes | **0.3693** | 0.2718 | **0.4101** | **0.5620** | | FB15k-237 | ULTRA-4g | Yes | 0.3684 | 0.2723 | 0.4051 | 0.5593 | | FB15k-237 | ULTRA-50g | Yes | 0.3675 | **0.2724** | 0.4037 | 0.5579 | | FB15k-237 | TRIX | Yes | 0.3618 | 0.2649 | 0.3989 | 0.5546 | | FB15k-237 | Flock | Yes | 0.3116 | 0.2215 | 0.3442 | 0.4912 | +| FB15k-237 | KG-ICL | Partial | 0.3310 | 0.2416 | 0.3626 | 0.5098 | | WN18RR | ULTRA-3g | Yes | 0.3691 | 0.2924 | 0.3923 | 0.5329 | | WN18RR | ULTRA-4g | Yes | 0.4838 | 0.4146 | 0.5198 | 0.6173 | | WN18RR | ULTRA-50g | Yes | 0.4282 | 0.3641 | 0.4600 | 0.5463 | | WN18RR | TRIX | Yes | 0.5065 | 0.4592 | 0.5217 | 0.6040 | | WN18RR | Flock | Yes | **0.5303** | **0.4783** | **0.5482** | **0.6367** | +| WN18RR | KG-ICL | No | 0.4387 | 0.4017 | 0.4515 | 0.5137 | | KINSHIP | ULTRA-3g | No | 0.2054 | 0.1439 | 0.2067 | 0.3156 | | KINSHIP | ULTRA-4g | No | 0.1298 | 0.0545 | 0.1336 | 0.2649 | | KINSHIP | ULTRA-50g | No | 0.2919 | 0.1858 | 0.2914 | 0.5442 | | KINSHIP | TRIX | No | 0.1487 | 0.1020 | 0.1373 | 0.2007 | | KINSHIP | Flock | No | **0.3803** | **0.2751** | **0.4013** | **0.6001** | +| KINSHIP | KG-ICL | No | 0.1522 | 0.0703 | 0.1480 | 0.3012 | | UMLS | ULTRA-3g | No | 0.6960 | 0.5983 | 0.7474 | 0.8956 | | UMLS | ULTRA-4g | No | 0.6684 | 0.5749 | 0.7118 | 0.8601 | | UMLS | ULTRA-50g | No | **0.8061** | **0.7330** | **0.8419** | **0.9607** | | UMLS | TRIX | No | 0.7256 | 0.6430 | 0.7632 | 0.8986 | | UMLS | Flock | No | 0.7768 | 0.7005 | 0.8169 | 0.9244 | +| UMLS | KG-ICL | No | 0.7797 | 0.7095 | 0.8132 | 0.9357 | | Countries-S1 | ULTRA-3g | No | 0.9375 | 0.8750 | **1.0000** | **1.0000** | | Countries-S1 | ULTRA-4g | No | **1.0000** | **1.0000** | **1.0000** | **1.0000** | | Countries-S1 | ULTRA-50g | No | 0.9757 | 0.9583 | **1.0000** | **1.0000** | | Countries-S1 | TRIX | No | 0.9271 | 0.8542 | **1.0000** | **1.0000** | | Countries-S1 | Flock | No | 0.9271 | 0.8542 | **1.0000** | **1.0000** | +| Countries-S1 | KG-ICL | No | 0.9132 | 0.8333 | **1.0000** | **1.0000** | | Countries-S2 | ULTRA-3g | No | 0.8715 | 0.7500 | **1.0000** | **1.0000** | | Countries-S2 | ULTRA-4g | No | **0.9167** | **0.8333** | **1.0000** | **1.0000** | | Countries-S2 | ULTRA-50g | No | 0.9062 | 0.8125 | **1.0000** | **1.0000** | | Countries-S2 | TRIX | No | 0.8854 | 0.7708 | **1.0000** | **1.0000** | | Countries-S2 | Flock | No | 0.8854 | 0.7708 | **1.0000** | **1.0000** | +| Countries-S2 | KG-ICL | No | 0.6643 | 0.5000 | 0.7708 | **1.0000** | | Countries-S3 | ULTRA-3g | No | 0.2354 | **0.0625** | 0.2917 | 0.6458 | | Countries-S3 | ULTRA-4g | No | 0.2459 | 0.0208 | 0.4375 | 0.5000 | | Countries-S3 | ULTRA-50g | No | 0.2049 | **0.0625** | 0.1667 | 0.6042 | | Countries-S3 | TRIX | No | **0.3625** | **0.0625** | **0.5833** | **0.8958** | | Countries-S3 | Flock | No | 0.2533 | 0.0208 | 0.4583 | 0.5000 | +| Countries-S3 | KG-ICL | No | 0.2455 | 0.0417 | 0.3958 | 0.5208 | † Original splits contain training/test leakage: h25/h50/h75 include 52/33/9 distinct test facts in training. Splits are preserved; these scores are not strictly held-out estimates. +KG-ICL uses prompt seed 0 and no answer-distance mask; its paper reports the official protocol, see the [KG-ICL guide](docs/kgicl.md#scores-and-the-official-evaluation-protocol). +
@@ -769,6 +855,10 @@ RTX 4070 Ti SUPER, float32, matched query batches, and five-repeat medians on sa \* Flock includes independently sampled walks at the same budget; identical-walk neural speedups are **1.23–2.01×**. YAGO3-10 Flock timings varied more. See the [full comparison and validation](docs/kgfm_inference.md). +KG-ICL, measured separately on an RTX 5070 Laptop GPU against the official code with its +[corrected defects](docs/kgicl.md#upstream-defects-corrected-in-dice), is **49.76×** faster on FB15k-237, +**4.43×** on WN18RR, and **35.58×** on YAGO3-10 ([details](docs/kgfm_inference.md#kg-icl)). +
## Link Prediction Benchmarks diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..233aa6387 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Benchmark harnesses; see cqa/ for complex query answering.""" diff --git a/benchmarks/adapters/kgicl_product_intersections.json b/benchmarks/adapters/kgicl_product_intersections.json new file mode 100644 index 000000000..3aafdc8fa --- /dev/null +++ b/benchmarks/adapters/kgicl_product_intersections.json @@ -0,0 +1,123 @@ +{ + "backbone_state_sha256": "8816c1555269384a6cd81a4b7650c9bd6e2394cec4cab5e7f75bd9909dcc9020", + "training": { + "objective": "filtered_softmax_v1", + "epochs": 500, + "batch_size": 8, + "learning_rate": 0.02, + "identity_penalty": 0.001, + "gradient_clip": 1.0, + "seed": 2026090851, + "weighting": "equal_source_shape", + "selection": "source_validation_mrr", + "shapes": [ + "2i", + "3i" + ], + "validation_sources": [ + "CoDExMedium", + "FB15k237", + "WN18RR" + ], + "validation_every": 5, + "training_sources": [ + "CoDExMedium", + "FB15k237", + "WN18RR" + ], + "train_per_shape": 140, + "early_stopping_patience": 100, + "beam_size": 64, + "tnorm": "prod", + "validation_shapes": [ + "1p", + "2p", + "3p", + "2i", + "3i", + "2in", + "3in", + "inp", + "pin", + "pni" + ], + "device": "cuda", + "validation_device": "cpu", + "cpu_threads": 4, + "training_cache_bytes": 6442450944, + "device_cache_bytes": 536870912, + "selected_epoch": 45, + "selected_validation_mrr": 0.18387722985219088, + "epochs_completed": 145, + "stopped_early": true, + "stop_reason": "validation_patience", + "training_queries": 840, + "optimizer_steps": 15225, + "row_cache": { + "FB15k237": { + "host_hits": 189328, + "device_hits": 92583, + "disk_rows": 0, + "scored_rows": 6581, + "feature_preparations": 6581, + "seconds": 3425.798498520162 + }, + "WN18RR": { + "host_hits": 185804, + "device_hits": 96153, + "disk_rows": 0, + "scored_rows": 6535, + "feature_preparations": 6535, + "seconds": 1409.8266095069703 + }, + "CoDExMedium": { + "host_hits": 186698, + "device_hits": 96815, + "disk_rows": 0, + "scored_rows": 4979, + "feature_preparations": 4979, + "seconds": 1814.797347703483 + } + } + }, + "score_banks": [ + "65bd769dade4744a953d7742ae96dee03c29d30075f9951792c8ac5b9df9bd3b", + "0fd86492f49259b3ab273d1885e0413b44389ae9507e7e32aaa96cb1a16b6c7e", + "279d5eff7128ad31d2fb5110f2d7d9d35a3b8959b7c14438322caae476a6c3ac" + ], + "sources": { + "FB15k237": "b144aa69f3fb06b4e62148adfae3f966c99776c8942ab987ec011d1a4855e6d1", + "WN18RR": "b5b638df29e8d26c20f4a32851d2266700a23d7bba4b28d84b917034a41bc9d1", + "CoDExMedium": "de72a1ae56a1f8ce0b54314c1cf6b8e06b0f65dba4fabf7ced49f725c2afcd6d" + }, + "version": 3, + "feature_mode": "context_scores", + "observed_mix": 1.0, + "bias_bound": 8.0, + "scale_bound": 2.0, + "normalization": "none", + "hidden_dim": 0, + "weights": [ + [ + -2.32368504798207, + 2.0049527287265114, + 1.4638496934559073, + -0.9412350507752393, + -0.260616153643281, + -1.8338308666061056, + -1.3255903255988397, + -0.6410547923533959 + ], + [ + -2.378670486469166, + -0.6121190155466186, + -0.773915400608249, + -1.6690557466596192, + -0.976340557301146, + -1.761396502127226, + -0.8284843834185431, + -1.0345695110208457 + ] + ], + "hidden_weights": null +} diff --git a/benchmarks/adapters/trix_product_intersections.json b/benchmarks/adapters/trix_product_intersections.json new file mode 100644 index 000000000..1e13c6c23 --- /dev/null +++ b/benchmarks/adapters/trix_product_intersections.json @@ -0,0 +1,122 @@ +{ + "backbone_state_sha256": "4a2e050130570df0964fc84cf14e6fcaec8672e30afb44f2b0aaf358864d95f7", + "training": { + "objective": "filtered_softmax_v1", + "epochs": 500, + "batch_size": 8, + "learning_rate": 0.02, + "identity_penalty": 0.001, + "gradient_clip": 1.0, + "seed": 2026090851, + "weighting": "equal_source_shape", + "selection": "source_validation_mrr", + "shapes": [ + "2i", + "3i" + ], + "validation_sources": [ + "CoDExMedium", + "FB15k237", + "WN18RR" + ], + "validation_every": 5, + "training_sources": [ + "CoDExMedium", + "FB15k237", + "WN18RR" + ], + "train_per_shape": 140, + "early_stopping_patience": 100, + "beam_size": 64, + "validation_shapes": [ + "1p", + "2p", + "3p", + "2i", + "3i", + "2in", + "3in", + "inp", + "pin", + "pni" + ], + "device": "cuda", + "validation_device": "cpu", + "cpu_threads": 4, + "training_cache_bytes": 6442450944, + "device_cache_bytes": 536870912, + "selected_epoch": 20, + "selected_validation_mrr": 0.21164792298119686, + "epochs_completed": 120, + "stopped_early": true, + "stop_reason": "validation_patience", + "training_queries": 840, + "optimizer_steps": 12600, + "row_cache": { + "FB15k237": { + "host_hits": 160251, + "device_hits": 72148, + "disk_rows": 0, + "scored_rows": 6353, + "feature_preparations": 6353, + "seconds": 3510.926437651331 + }, + "WN18RR": { + "host_hits": 156886, + "device_hits": 75333, + "disk_rows": 0, + "scored_rows": 6533, + "feature_preparations": 6533, + "seconds": 1340.7423292564345 + }, + "CoDExMedium": { + "host_hits": 157168, + "device_hits": 76657, + "disk_rows": 0, + "scored_rows": 4927, + "feature_preparations": 4927, + "seconds": 1591.4698363094358 + } + } + }, + "score_banks": [ + "8d06f4ecca4f7c446ed706644f784b789c48cc7c0797df173915ea99b158379b", + "6e8c83f950fb39daae2e43f662e3384727b42dfc69250b851bdadf64f396021b", + "1981e87027c19d29146a1ed70c9c9ecbb47382d45cb73aef8b326bf7b4cc1fd6" + ], + "sources": { + "FB15k237": "b144aa69f3fb06b4e62148adfae3f966c99776c8942ab987ec011d1a4855e6d1", + "WN18RR": "b5b638df29e8d26c20f4a32851d2266700a23d7bba4b28d84b917034a41bc9d1", + "CoDExMedium": "de72a1ae56a1f8ce0b54314c1cf6b8e06b0f65dba4fabf7ced49f725c2afcd6d" + }, + "version": 3, + "feature_mode": "context_scores_v1", + "observed_mix": 1.0, + "bias_bound": 8.0, + "scale_bound": 2.0, + "normalization": "none", + "hidden_dim": 0, + "weights": [ + [ + -0.8248981279689328, + 1.3857730002558575, + 1.2953982008249219, + -0.4644471807543671, + 0.10034981678214763, + 1.0685651164671832, + -0.9667501449968788, + 0.5089790944483673 + ], + [ + -1.0692246624707495, + 1.853212091799282, + 0.38918544229708163, + -0.13628156778504116, + -0.8493974313750033, + -0.8462766593623741, + -1.072134991578466, + -0.28842092871795827 + ] + ], + "hidden_weights": null +} diff --git a/benchmarks/adapters/ultra_product_intersections.json b/benchmarks/adapters/ultra_product_intersections.json new file mode 100644 index 000000000..14b2cdd3b --- /dev/null +++ b/benchmarks/adapters/ultra_product_intersections.json @@ -0,0 +1,91 @@ +{ + "backbone_state_sha256": "5524f131f909918c40daa0c6169eaf8ebb7ca344ccf9e96e1d81154cd5097e5b", + "training": { + "objective": "filtered_softmax_v1", + "epochs": 500, + "batch_size": 8, + "learning_rate": 0.02, + "identity_penalty": 0.001, + "gradient_clip": 1.0, + "seed": 2026090851, + "weighting": "equal_source_shape", + "selection": "source_validation_mrr", + "shapes": [ + "2i", + "3i" + ], + "validation_sources": [ + "CoDExMedium", + "FB15k237", + "WN18RR" + ], + "validation_every": 5, + "training_sources": [ + "CoDExMedium", + "FB15k237", + "WN18RR" + ], + "train_per_shape": 140, + "early_stopping_patience": 100, + "beam_size": 64, + "validation_shapes": [ + "1p", + "2p", + "3p", + "2i", + "3i", + "2in", + "3in", + "inp", + "pin", + "pni" + ], + "selected_epoch": 20, + "selected_validation_mrr": 0.2080190892168318, + "epochs_completed": 120, + "stopped_early": true, + "stop_reason": "validation_patience", + "training_queries": 840, + "optimizer_steps": 12600 + }, + "score_banks": [ + "7f7701967dc5e2a9c11c0283d4cc73bd14bf4dc07472c10550d3ceb57e49e39f", + "4e888aacfb61da3e6dd95f6e81d51bf4de064873ec9e09986d3ed22a153907ce", + "19712b80292de03ff8ed1fda781c4f6691b4b05817b8c7e74a0075d1de78aa9b" + ], + "sources": { + "FB15k237": "b144aa69f3fb06b4e62148adfae3f966c99776c8942ab987ec011d1a4855e6d1", + "WN18RR": "b5b638df29e8d26c20f4a32851d2266700a23d7bba4b28d84b917034a41bc9d1", + "CoDExMedium": "de72a1ae56a1f8ce0b54314c1cf6b8e06b0f65dba4fabf7ced49f725c2afcd6d" + }, + "version": 3, + "feature_mode": "context_scores_v1", + "observed_mix": 1.0, + "bias_bound": 8.0, + "scale_bound": 2.0, + "normalization": "none", + "hidden_dim": 0, + "weights": [ + [ + -0.9610871975006661, + 1.6151930713458116, + 1.506114487842482, + -0.24227659844007385, + 0.7539009660458432, + 1.241735976991373, + 0.6837330334025685, + 0.6785526796481871 + ], + [ + -0.6606956373109719, + 1.7630503354217026, + 0.31905344378446204, + -0.061247942677854626, + -0.46463820640491177, + -1.14732100440598, + -0.6411935744153302, + -0.09846041230646553 + ] + ], + "hidden_weights": null +} diff --git a/benchmarks/cqa/Dockerfile b/benchmarks/cqa/Dockerfile new file mode 100644 index 000000000..84da3ac70 --- /dev/null +++ b/benchmarks/cqa/Dockerfile @@ -0,0 +1,37 @@ +FROM python:3.11.14-slim-bookworm@sha256:65a93d69fa75478d554f4ad27c85c1e69fa184956261b4301ebaf6dbb0a3543d AS runtime +RUN rm /etc/apt/sources.list.d/debian.sources \ + && echo 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/20260901T000000Z bookworm main' > /etc/apt/sources.list \ + && apt-get -o Acquire::Retries=5 update \ + && apt-get install -y --no-install-recommends g++ \ + && rm -rf /var/lib/apt/lists/* + +FROM runtime AS dependencies + +COPY benchmarks/cqa/requirements.lock /tmp/requirements.lock +COPY wheels /wheels +RUN sed 's|torch @ https://[^ ]*|torch==2.9.1+cu128|' /tmp/requirements.lock > /tmp/offline.lock \ + && pip install --no-cache-dir --no-index --find-links=/wheels --require-hashes -r /tmp/offline.lock \ + && pip check + +FROM runtime +COPY --from=dependencies /usr/local /usr/local +WORKDIR /opt/dicee +COPY dicee dicee +COPY pyproject.toml pyproject.toml +COPY benchmarks/__init__.py benchmarks/__init__.py +COPY benchmarks/cqa benchmarks/cqa +COPY benchmarks/adapters benchmarks/adapters +COPY benchmarks/plus_h benchmarks/plus_h +COPY benchmarks/ultraquery benchmarks/ultraquery +ARG SOURCE_COMMIT=unknown +ARG SOURCE_DIRTY=unknown +LABEL org.opencontainers.image.revision=$SOURCE_COMMIT +ENV DICEE_SOURCE_COMMIT=$SOURCE_COMMIT \ + DICEE_SOURCE_DIRTY=$SOURCE_DIRTY \ + PYTHONPATH=/opt/dicee \ + PYTHONHASHSEED=0 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + CUBLAS_WORKSPACE_CONFIG=:4096:8 \ + PYTORCH_ALLOC_CONF=expandable_segments:True +ENTRYPOINT ["python", "-m", "benchmarks.cqa"] diff --git a/benchmarks/cqa/PROTOCOL.md b/benchmarks/cqa/PROTOCOL.md new file mode 100644 index 000000000..33a73198f --- /dev/null +++ b/benchmarks/cqa/PROTOCOL.md @@ -0,0 +1,156 @@ +# Evaluation protocol + +This document fixes how the [benchmark suites](README.md) score methods, what +verification establishes, and where this reproduction knowingly differs from a +publication. Where the evidence for a published number is uncertain, the +difference is recorded rather than resolved by tuning. + +## Metrics + +Each query has easy answers, which follow from the observed graph, and hard +answers, which need at least one missing link. MRR and Hits@k rank every hard +answer against all candidate entities after filtering every other easy and hard +answer. Scores average hard answers per query, queries per query type, and +query types equally within a dataset group; suite means weight datasets +equally. Reports state whether a run covers the full split and every query +type of its dataset, and partial runs are never presented as full-suite scores. + +Ties are scored twice from the same predictions: `sort` follows PyTorch's +default descending argsort, as the reference evaluator does. That sort is not +guaranteed stable, so exactly tied scores can order differently on another +device or library version. `expected` is the exact expectation of each rank +under uniformly random ordering within a tied block, which removes that +dependence. `expected` is not the midpoint rank. Rank traces store every hard answer's ranks under both policies, +so later analyses never rerun inference. + +Each suite keeps its released candidates and inference graphs. UltraQuery +evaluates its 14 query types on the published inference graphs: the training +graph for transductive datasets and the released inference graphs for the +inductive ones. Test triples never enter inference in either suite. + +## +H answer filters + +The `benchs-1.0` archive has two sets of test answers for the same queries: the +root `test-{easy,hard}-answers.pkl` files and the `test-query-reduction//all/` +files, which the pinned +[analysis script](https://github.com/april-tools/is-cqa-complex/blob/d1ce74164936a7c09d9147e83190da047cb39429/read_queries_pair.py) +recomputes from the graph: + +| Dataset | Type | Root hard answers | Reduction `all` hard answers | +|---|---|---:|---:| +| FB15k-237+H | 3in | 20,000 | 192,617 | +| FB15k-237+H | pin | 20,000 | 197,728 | +| FB15k-237+H | inp | 20,000 | 201,328 | +| NELL995+H | 3in | 20,000 | 126,893 | +| NELL995+H | pin | 20,000 | 133,156 | +| NELL995+H | inp | 20,000 | 128,812 | + +For `2in` and `pni` the hard answers agree, but the root easy-answer filters +are wrong for every FB15k-237+H and NELL995+H query: 18,000 queries with 149,691 +false filtered answers in total. All hard answers are true, and the other +negated types and ICEWS18+H agree with full-graph truth. The pinned authors' +`achieve_answer` gives the same result on all 52,413 checked queries. + +The **corrected** filter, the +H default, takes the easy answers of `2in` and +`pni` on FB15k-237+H and NELL995+H from the authors' `2in/all` and `pni/all` +files and keeps every root hard answer. Preparation checks those files against +the complete graph and pins them in the study. Every prediction is also scored +with the **released** filters as a control. Test facts are used only for this +offline audit. On all FB15k-237+H `2in` queries, a CPU QTO diagnostic with +train+valid facts scores 16.11 MRR with released and 10.59 with corrected +filters, against the published 10.6; with training facts only, 13.16 and 9.62. + +## +H inference graphs + +Graph-based methods (GNN-QE, UltraQuery, QTO, CQD-Hybrid, ULTRA and TRIX) +answer test queries over train+valid facts by default, the observed graph from +which the released test difficulty groups were built. Validation always uses +training facts. The released GNN-QE, UltraQuery and QTO loaders use training +facts for these transductive graphs, and the +H CQD-Hybrid loader uses +train+valid for FB15k-237+H and NELL995+H but skips observed filters for +ICEWS18+H. Both graph conditions can be compared directly (see the README's +ablations); reference metadata records each upstream loader's own setting. + +## Operators + +[Appendix F](https://arxiv.org/html/2410.12537v3#A6) of the +H paper specifies +the minimum t-norm for FB15k-237+H `2u`. With it, full-query CPU diagnostics +give CQD 27.77 and CQD-Hybrid 27.77 MRR; with the product t-norm, 40.05 and +42.17, against the published 40.1 and 42.2. The recipes therefore use product +for this query type. This is an inferred documentation mismatch, not proof of +the original command. The substitution does not explain `up`: product gives +7.88 (CQD) and 9.33 (CQD-Hybrid), below the minimum t-norm's 9.15 and 11.06 and +the published 10.6 and 12.0, so `up` keeps the documented minimum and remains +unreproduced. + +## Verification + +Every published method is a native port. Verification runs the frozen +validation batches through the pinned upstream code and the port with identical +inputs, and requires scores within tolerance (absolute and relative 3e-5) and +identical candidate order and ties for every query type. Reference graph hashes +must match both validation and test graphs. Upstream batch composition is +preserved: methods whose released code batches queries (ConE, CLMPT, GNN-QE, +UltraQuery) receive the same reference batches. + +Verification establishes the port under the declared inputs. It does not +establish the unpublished command behind every paper score; small remaining +differences, such as NELL995+H CLMPT, are not attributed to the authors without +evidence. For ULTRA and TRIX, an independent implementation recomputes context +features, calibration, pruning, composition and identity controls from captured +backbone logits; backbone checkpoint parity is tested separately. + +The bounded CQD profile is verified against an equally bounded reference and +can differ numerically from dense upstream execution. + +Two opt-in variants are not upstream +methods. UltraQuery with `observed_traversal` also takes, at every projection, +the maximum with an exact traversal of the observed facts from the same fuzzy +set; its oracles come from the pinned upstream code with that maximum added +through upstream's own `SymbolicTraversal`. UltraQuery's released weights used +as a frozen ULTRA backbone (`relation_conditioning: query`) condition relation +reasoning on the queried relation, as UltraQuery's projection does; ULTRA's link +prediction conditions an inverse tail query on its direct relation instead. +With that setting the frozen backbone equals UltraQuery's single-source +projection for every relation (tests/test_query_baselines.py). + +## Reproducibility + +Inference is float32 with IEEE matrix products, no autocast and deterministic +algorithms. Message passing in ULTRA, TRIX, UltraQuery and GNN-QE reduces each +node's edges in a fixed order, so scores do not depend on thread scheduling. +Studies pin the input files, query batches, source code, runtime image and GPU +model, and refuse to run when any of them changes. Resuming a study requires +the same identity; completed entries are never recomputed. + +Matrix products in the backbones are not batch-invariant: the same query can +receive different last bits in a different atomic batch. The KGFM adapters +reuse backbone rows through caches, so cache budgets, batch sizes and the +point at which an interrupted run resumes with empty caches can change the +last bits of later scores. Batch and cache settings are therefore frozen with +each study, verified with exact tie checks, and changed only in a new study. +A batch-invariant backbone would remove this dependence. + +## Answer-level difficulty + +`difficulty-report` assigns each +H hard answer its minimum number of missing +positive links over full-graph witnesses, relative to train+valid facts for +every method. Intersections add costs, unions take the cheapest branch, and +negated branches restrict truth without adding cost, following the paper's +positive reasoning trees; negative-edge difficulty is not measured. Scores +average answers within participating queries, then queries within the parent +type; no average mixes parent types. + +The authors' released reductions are relative to train+valid, as their pinned +[generator](https://github.com/april-tools/is-cqa-complex/blob/d1ce74164936a7c09d9147e83190da047cb39429/create_queries.py) +shows. They are reported separately, and their union reductions are not +translated into link counts. + +## Upstream evaluation edge case + +UltraQuery's `batch_evaluate` masks excluded candidates with `-inf`, yet can +still count them ahead of a hard answer whose score is also `-inf`: with six +`-inf` scores, candidates `[0, 2, 4]` and hard answer 2, it returns rank 3 +where the candidate-only rank is 2. The shared evaluator ranks candidates only. +Native UltraQuery returns finite logits, so this does not affect its published +results, and finite-score parity remains required. diff --git a/benchmarks/cqa/README.md b/benchmarks/cqa/README.md new file mode 100644 index 000000000..bf4d6b028 --- /dev/null +++ b/benchmarks/cqa/README.md @@ -0,0 +1,227 @@ +# Complex query answering benchmarks + +Reproducible evaluation of complex query answering (CQA) on two suites: + +| Suite | Datasets | Query types | Answer filters | +|---|---|---|---| +| `ultraquery` | The 23 released UltraQuery datasets: 3 transductive, 9 inductive with new entities, 11 WikiTopics with new entities and relations | 14 (9 positive, 5 negated) | released | +| `plus_h` | **+H**: FB15k237+H, NELL995+H and ICEWS18+H from [Gregucci et al.](https://arxiv.org/abs/2410.12537), whose hard answers are balanced across the number of links that must be predicted to reach them | 16 (adds `4p`, `4i`) | corrected, with released controls | + +Each suite evaluates published methods (UltraQuery, GNN-QE, QTO, CQD, CQD-Hybrid, +ConE, CLMPT) through native DICE ports and the ULTRA and TRIX foundation models +with learned score adapters, each paired with an identity-adapter control. Every +prediction is scored under two tie policies: sort order, and the exact +expectation under random ordering of tied scores. + +A study freezes its inputs, query batches and source code, checks validation +predictions against the pinned upstream implementations, and only then runs +the test split. [PROTOCOL.md](PROTOCOL.md) states the evaluation protocol and +every known deviation from the publications. + +## Quick start + +Everything runs through one command line from the repository root: + +```bash +python -m benchmarks.cqa {plus_h,ultraquery} COMMAND [options] +``` + +Commands run in the current environment, which needs DICE and PyTorch. With +`--image IMAGE` they run in the pinned Docker runtime instead; the host then +needs only Python 3, Docker and, for GPUs, the NVIDIA Container Toolkit. +Containers have no network access, read inputs from `--input-root` (default: +the repository) read-only, and write only below `--output`; `--gpu cdi` +selects CDI device access and `--gpu none` runs on CPU. + +Exploratory evaluation of any subset needs no preparation. `--split` is +required, so the test split is only ever evaluated on request: + +```bash +python -m benchmarks.cqa plus_h evaluate --split valid --output results/cqd-negation \ + --methods cqd cqd-hybrid --datasets FB15k237+H --query-types negation +``` + +The full verified workflow for one suite: + +```bash +export IMAGE=dicee/cqa:local STUDY="$PWD/results/plus-h" +python -m benchmarks.cqa plus_h build --image "$IMAGE" # one image serves both suites +python -m benchmarks.cqa plus_h prepare --image "$IMAGE" --output "$STUDY" +# Export one validation oracle per published-method entry (see Verification). +python -m benchmarks.cqa plus_h verify --image "$IMAGE" --output "$STUDY" --references "$PWD/references" +python -m benchmarks.cqa plus_h run --image "$IMAGE" --output "$STUDY" +python -m benchmarks.cqa plus_h report --image "$IMAGE" --results "$STUDY/test" --output "$STUDY/reports" +``` + +| Command | Purpose | +|---|---| +| `setup` | Download the selected datasets and released checkpoints and install the shipped adapters; `--check` only reports missing files and checksum mismatches | +| `evaluate` | Direct evaluation of recipe subsets, without a frozen study | +| `prepare` | Resolve recipes and freeze inputs, query batches and source in `STUDY/bundle` | +| `verify` | Check every entry against its oracle in a fresh process and pin the evidence in `STUDY/verified-bundle` | +| `run` | Run the test split of a verified study into `STUDY/test`, or a validation pilot into `STUDY/pilot` with `--phase pilot` | +| `report` | Tables, paired adapter/filter/graph effects and comparisons with published scores, from completed results | +| `difficulty-report` | Answer-level difficulty of completed +H results, on CPU | +| `train` | Train the UltraQuery comparison baselines with pinned author code | +| `build` | Build the runtime image, or with `--training BASE_IMAGE` the CPU author-training image | + +`setup` runs on the host with only the standard library. `evaluate`, `prepare` +and `run` call it first; `--no-setup` instead requires existing inputs. +Downloads are streamed with bounded buffers and disk-space checks, released +checkpoints are checked by SHA-256, and existing files are never replaced. If +the +H authors' model host rejects automated downloads, pass +`--models-archive iscqa-compl-models.zip`. `--dry-run` prints resolved recipes, +input sources or the Docker command without reading inputs or writing files. + +## Recipes and selection + +A recipe fixes method, checkpoint, dataset, query types, query batches and +execution options. Each suite directory holds its public recipes: + +| File | Contents | +|---|---| +| `plus_h/baselines.json` | The seven published methods on the three +H datasets (21 entries) | +| `plus_h/kgfm_adapters.json` | ULTRA and TRIX with `2i`/`3i`-trained adapters (6 entries, each with an identity control) | +| `plus_h/kgfm_kgicl.json` | KG-ICL with the product adapter (3 entries, each with an identity control) | +| `plus_h/published_results.json` | Scores reported by the +H authors, for `report` | +| `ultraquery/baselines.json` | Native UltraQuery on all 23 datasets | +| `ultraquery/kgfm_adapters.json` | ULTRA and TRIX with `2i`/`3i`-trained adapters on all 23 datasets | +| `ultraquery/kgfm_kgicl.json` | KG-ICL with the product adapter on all 23 datasets (each with an identity control) | +| `ultraquery/comparisons.json` | ULTRA link-prediction weights, an incoming-relation heuristic and QTO | +| `ultraquery/trained_baselines.json` | QTO on FB15k and inductive GNN-QE, after `train` | + +Defaults are `baselines.json` and `kgfm_adapters.json`; `--manifests FILE ...` +replaces them. The KG-ICL recipes are never defaults: they list their blockers +and need a fitted, pinned adapter before any test run ([KG-ICL guide](../../docs/kgicl.md#complex-query-answering)). Adapter weights shared by both +suites are in [`benchmarks/adapters`](../adapters). Recipe files list shared `defaults` once +per method, and each entry only what differs; `{dataset}` in an ID is filled in. +[`tests/recipe_fingerprints.json`](tests/recipe_fingerprints.json) pins every +resolved entry, so a recipe change is always visible in review. + +Selectors combine freely and apply to every command that reads recipes: + +- `--methods`, `--datasets`, `--entries`: omit them or pass `all` for every recipe. +- `--query-types`: individual types and the groups `epfo`, `negation` and `all`, + for example `--query-types epfo 2in`. Omit it to keep each recipe's coverage. +- `--split valid` and `--max-queries-per-shape N` give reproducible samples. + +The +H CQD and CQD-Hybrid recipes cover all 16 types, using signed-atom (CQD-A) +negation with +H scoring; `--atomic-negation` enables it for custom recipes. +Query types and execution options are chosen when a study is prepared and +cannot change on `run`; `run` accepts method, dataset and entry selectors +within the frozen study. Repeating a command resumes completed work. + +## Verification + +`verify` compares each published method's validation predictions with an +oracle exported from its pinned upstream checkout (`REFERENCES` in +[`dicee/query_answering/catalog.py`](../../dicee/query_answering/catalog.py)). +Export one oracle per entry in an environment with the authors' dependencies, +matching Python, PyTorch, CUDA, GPU, driver, precision and threads: + +```bash +python benchmarks/cqa/verification/export_reference.py \ + --bundle "$STUDY/bundle" --input-root "$PWD" --entry cqd-FB15k237+H \ + --upstream "$UPSTREAM_CQD" --device cuda --output references/cqd-FB15k237+H.pt +``` + +The exporter imports no DICE code. Acceptance needs scores within tolerance and +identical candidate order and ties for every query. CQD and CQD-Hybrid answer +the five negated +H types with CQD-A signed-atom negation, which the pinned +H +code does not implement: their oracles cover the eleven positive types, and +`verify` records the negated types as evaluated without an upstream +reference; fixtures check them against CQD-A outputs. ULTRA/TRIX entries need no +oracle: an independent implementation checks calibration, pruning, composition +and identity controls on captured backbone logits; the backbones' own parity is +tested separately. Passing evidence of an unchanged entry is reused (an +ULTRA/TRIX integration check, or a parity check against the same oracles), so +an interrupted `verify` resumes and checks can run as separate jobs; an +incomplete ULTRA/TRIX check must be removed from `STUDY/verification` first. Evidence binds inputs, query batches, source, image and +hardware; a change to any of them needs a new study. For the bounded profile, +`--comparison-references` additionally compares CQD with dense oracles. + +The default `bounded` profile runs CQD and CQD-Hybrid in 8 GB of GPU memory +(`row_batch_size` and `final_batch_size` 32, no reference batching) with +unchanged beams; `--profile reference` keeps dense upstream batching. Oracles +must come from a study prepared with the same profile. + +## Ablations + +Each ablation is a separate study with its own verification. + +- **Inference graph** (+H): `prepare --inference-graphs train train+valid`. + Graph-based methods use train+valid facts by default; ConE, CLMPT and CQD do + not read the graph and run once. IDs gain `-graph-train` or `-graph-train-valid`. +- **Observed facts** (ULTRA/TRIX): `prepare --observed-facts none atomic` or + `all`, with unchanged adapter weights. IDs gain `-facts-MODE`. +- **Answer filters** (+H): every prediction is also scored with the released + filters (`-released-filters`); `--answer-filter released` prepares a study + with released filters only. +- **Adapters**: every shipped ULTRA/TRIX entry has a `without-adapter` control + scored from the same backbone batches. + +`report` writes paired effects with query-bootstrap confidence intervals +conditional on the frozen weights: `adapter-effects`, `filter-effects` and +`graph-effects`. + +## Hardware and parallelism + +`--hardware-profile h100` (on `prepare` or `evaluate`) is the configuration +measured on full validation splits on H100 80 GB and H100 NVL GPUs: ULTRA/TRIX +use atomic batches of 16, a 512 MiB budget for calibrated scores, and a 24 GiB +cache of raw backbone scores in host memory, kept per row so that the beams of +different queries share it. `--kgfm-batch-size N` overrides the batch. Other +methods keep their settings. These settings can change floating-point results, +so they are frozen with the study and verified on the target GPU. + +`run` and `evaluate` accept `--gpus 0 1 2 3`: each entry runs in a fresh worker +that sees one GPU, longest methods first, with results identical to a +sequential run; `verify` runs its checks the same way. `--workers-per-gpu 2` +overlaps one worker's host work with another's GPU work, for about 1.5 times +the throughput of one worker; with the H100 profile each KGFM worker stays +below 12 GiB of GPU memory and holds up to 24 GiB of host memory. Keep one +worker per GPU on GPUs shared with other jobs. + +The QTO recipes build each relation's thresholded calibrated matrix once, from +the released canonical 100-head blocks, and keep it in CSR form: up to 6 GiB +on the GPU (`matrix_bytes`) and 32 GiB in host memory (`host_matrix_bytes`), +least recently used out first. Scores are identical to computing the rows for +every query, which recomputes almost a whole relation matrix for each chained +projection and needs days per full test split. + +## UltraQuery comparison baselines + +`trained_baselines.json` needs weights trained locally with pinned author code +(QTO and InductiveQE checkouts below `Experiments/query-baselines/upstream` in +the input root). Training defaults to CPU and the authors' hyperparameters, +selects checkpoints on validation MRR only, and records code revisions, inputs, +environment and checkpoint hashes in `training.json`: + +```bash +python -m benchmarks.cqa ultraquery build --image "$IMAGE-training" --training "$IMAGE" +python -m benchmarks.cqa ultraquery train --image "$IMAGE-training" \ + --output "$PWD/checkpoints/ultraquery_benchmark-trained" --methods qto inductive-gnnqe +``` + +Each job also writes a `manifest.json` for its checkpoint; at the default +output path above, `trained_baselines.json` finds the checkpoints directly. + +Inductive GNN-QE requires the complete corrected v2.0 InductiveQE archives +([Zenodo 7306046](https://zenodo.org/records/7306046)); every file is checked +against [`inductive-v2-files.json`](../ultraquery/inductive-v2-files.json). +`--smoke` trains a tiny model for one update, and final runs reject it. The +author runtime applies three mechanical TorchDrug build fixes +([`training/torchdrug_compat.py`](training/torchdrug_compat.py)) without +changing kernel arithmetic. + +## Tests + +```bash +python -m pytest benchmarks +``` + +runs the harness and protocol tests on CPU without datasets. Tests that +need upstream checkouts or released data are skipped unless configured: +`DICEE_CQD_REFERENCE`, `DICEE_ULTRAQUERY_REFERENCE_ROOT`, +`DICEE_INDUCTIVE_REFERENCE_ROOT` and `DICEE_ULTRAQUERY_DATA_ROOT` point to the +pinned checkouts and the extracted UltraQuery data. diff --git a/benchmarks/cqa/__init__.py b/benchmarks/cqa/__init__.py new file mode 100644 index 000000000..b647dec1b --- /dev/null +++ b/benchmarks/cqa/__init__.py @@ -0,0 +1,4 @@ +"""Reproducible complex-query benchmarks: +H and UltraQuery. + +Run ``python -m benchmarks.cqa --help``. +""" diff --git a/benchmarks/cqa/__main__.py b/benchmarks/cqa/__main__.py new file mode 100644 index 000000000..4e28416e1 --- /dev/null +++ b/benchmarks/cqa/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +main() diff --git a/benchmarks/cqa/cli.py b/benchmarks/cqa/cli.py new file mode 100644 index 000000000..575320396 --- /dev/null +++ b/benchmarks/cqa/cli.py @@ -0,0 +1,517 @@ +"""Prepare, verify, run and report the reproducible complex-query benchmarks. + + python -m benchmarks.cqa {plus_h,ultraquery} COMMAND [options] + +Commands run in this process, which needs dicee and PyTorch, or with ``--image`` +inside the pinned Docker runtime; the host then needs only Python 3 and Docker. +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +from . import docker, inputs +from .manifests import REPO, SUITES, default_recipes, prepare_manifest, read_manifest, select_entries, suite_directory + +# Longest expected runtimes start first, so short entries fill the remaining slots. +RUNTIME_ORDER = ('trix-adapter', 'ultra-adapter', 'gnnqe', 'cqd-hybrid', 'cqd', 'qto') +# Container GPU access per command; other commands always run on the host. +CONTAINER_GPU = {'evaluate': 'runtime', 'prepare': 'none', 'verify': 'runtime', 'run': 'runtime', + 'report': 'none', 'difficulty-report': 'none', 'train': 'none'} +HOST_OPTIONS = {'image', 'gpu', 'dry_run', 'models_archive', 'input_root', 'output', 'no_setup'} + + +def write_json(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2) + '\n') + + +def build_parser(): + parser = argparse.ArgumentParser(prog='python -m benchmarks.cqa', description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('suite', choices=SUITES) + commands = parser.add_subparsers(dest='command', required=True) + add = {} + + def command(name, help, *, hidden=False): + add[name] = commands.add_parser(name, help=argparse.SUPPRESS if hidden else help, description=help) + add[name].add_argument('--input-root', type=Path, default=REPO, help='Root of datasets and checkpoints (default: repository)') + return add[name] + + def selection(p, *, query_types=True): + p.add_argument('--entries', nargs='+', action='extend', help='Entry IDs') + p.add_argument('--methods', nargs='+', action='extend', help='Methods; omit or use all for every recipe') + p.add_argument('--datasets', nargs='+', action='extend', help='Datasets; omit or use all for every recipe') + if query_types: + p.add_argument('--query-types', nargs='+', action='extend', help='Query types or groups: all, epfo, negation') + p.add_argument('--atomic-negation', action='store_true', help='CQD/CQD-Hybrid signed-atom negation for custom recipes') + + def recipes(p, *, ablations=False): + p.add_argument('--manifests', type=Path, nargs='+', action='extend', help='Recipe manifests (default: public recipes)') + p.add_argument('--profile', choices=('bounded', 'reference'), default='bounded', + help='CQD batching: bounded (8 GB GPUs) or dense reference batches') + p.add_argument('--hardware-profile', choices=('default', 'h100'), default='default', + help='KGFM batch/cache starting point; requires fresh verification') + p.add_argument('--kgfm-batch-size', type=int, help='Atomic row and backbone batch of KGFM adapters') + p.add_argument('--answer-filter', choices=('released', 'corrected'), + help='+H default: corrected with released controls; UltraQuery: released only') + if ablations: + p.add_argument('--observed-facts', nargs='+', action='extend', choices=('none', 'atomic', 'both', 'all'), + help='KGFM observed-fact ablations with unchanged adapter weights') + p.add_argument('--inference-graphs', nargs='+', action='extend', choices=('train', 'train+valid'), + help='+H only: test inference graphs (default: train+valid)') + + def inputs_options(p): + p.add_argument('--no-setup', action='store_true', help='Require existing inputs instead of downloading them') + p.add_argument('--models-archive', type=Path, help="Official +H weights ZIP if the authors' host is unavailable") + + def container(p): + p.add_argument('--image', help='Run inside this pinned Docker image (tag or image ID)') + p.add_argument('--gpu', choices=('runtime', 'cdi', 'none'), help='Docker GPU access') + p.add_argument('--dry-run', action='store_true', help='Print the command without running it') + + def parallel(p): + p.add_argument('--gpus', nargs='+', help='Run entries concurrently, one listed GPU (index or UUID) per worker') + p.add_argument('--workers-per-gpu', type=int, default=1, help='Concurrent entries per GPU (default: 1)') + + p = command('setup', 'Download missing datasets and checkpoints and install shipped adapters; no inference') + recipes(p, ablations=True) + selection(p) + p.add_argument('--models-archive', type=Path, help="Official +H weights ZIP if the authors' host is unavailable") + p.add_argument('--check', action='store_true', help='Only report missing inputs and checksum mismatches') + p.add_argument('--dry-run', action='store_true', help='Print sources without downloading') + + p = command('evaluate', 'Evaluate recipe subsets directly, without frozen-study verification') + recipes(p) + selection(p) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--device', default='cuda') + p.add_argument('--split', choices=('valid', 'test'), required=True, + help='Query split; required, so the test split is only ever evaluated on request') + p.add_argument('--max-queries-per-shape', type=int, help='Uniform sample of each query type') + p.add_argument('--seed', type=int) + p.add_argument('--threads', type=int) + inputs_options(p) + parallel(p) + container(p) + + p = command('prepare', 'Resolve recipes and freeze inputs, query batches and source in OUTPUT/bundle') + recipes(p, ablations=True) + selection(p) + p.add_argument('--output', type=Path, required=True, help='Study directory, below the input root for Docker') + inputs_options(p) + container(p) + + p = command('verify', 'Check validation parity and pin the evidence in OUTPUT/verified-bundle') + p.add_argument('--output', type=Path, required=True, help='Study directory containing bundle/') + p.add_argument('--references', type=Path, help='Directory of exported baseline oracles, REFERENCES/ENTRY.pt') + p.add_argument('--comparison-references', type=Path, help='Optional dense CQD oracles for comparison reporting') + p.add_argument('--device', default='cuda') + p.add_argument('--study-path', help=argparse.SUPPRESS) + parallel(p) + container(p) + + p = command('run', 'Run frozen entries: a validation pilot, or the test after verification') + p.add_argument('--output', type=Path, required=True, help='Study directory; results go to OUTPUT/PHASE') + selection(p, query_types=False) + p.add_argument('--phase', choices=('pilot', 'test'), default='test', help='pilot uses bundle/, test verified-bundle/') + p.add_argument('--device', default='cuda') + p.add_argument('--pilot-queries', type=int, default=2, help='Minimum queries per type, rounded up to whole reference batches') + p.add_argument('--worker', action='store_true', help=argparse.SUPPRESS) + inputs_options(p) + parallel(p) + container(p) + + p = command('report', 'Recompute tables and paired deltas from completed results; no inference') + p.add_argument('--results', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--published', type=Path, help="Published scores (default: the suite's published_results.json)") + p.add_argument('--bootstrap-samples', type=int, default=2000) + p.add_argument('--seed', type=int, default=0) + p.add_argument('--workers', type=int, default=1, help='Processes for traces and effects; results do not depend on it') + container(p) + + p = command('difficulty-report', 'Answer-level difficulty from completed frozen rank traces; CPU only') + p.add_argument('--results', type=Path, required=True) + p.add_argument('--bundle', type=Path, required=True) + p.add_argument('--output', type=Path, required=True, help='New directory outside frozen results and bundles') + p.add_argument('--labels-root', type=Path, help='Separate extraction root for released reduction labels') + p.add_argument('--entries', nargs='+', action='extend', help='Completed result IDs, including control suffixes') + p.add_argument('--query-types', nargs='+', action='extend', help='Report-only query-type subset') + p.add_argument('--direct', action='store_true', + help='Read results of direct evaluate runs; the bundle only pins the data files and filter corrections') + container(p) + + p = command('train', 'Train UltraQuery baselines in the author runtime (UltraQuery suite)') + from .train import add_arguments + add_arguments(p) + container(p) + + p = command('build', 'Build the pinned Docker runtime, or the CPU author-training runtime with --training') + p.add_argument('--image', required=True, help='Tag for the new image') + p.add_argument('--training', metavar='BASE_IMAGE', help='Build the training runtime on top of this benchmark image') + p.add_argument('--cache', type=Path, help='Wheel cache (default: ~/.cache/dicee/cqa)') + p.add_argument('--network', default='default', help='Use host if Docker bridge DNS is unavailable') + + p = command('parity', 'Verify one baseline against its exported oracle', hidden=True) + p.add_argument('--bundle', type=Path, required=True) + p.add_argument('--entry', required=True) + p.add_argument('--reference', type=Path, required=True) + p.add_argument('--comparison-reference', type=Path) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--device', default='cuda') + + p = command('integration', 'Verify one KGFM adapter pipeline against the independent reference', hidden=True) + p.add_argument('--bundle', type=Path, required=True) + p.add_argument('--entry', required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--device', default='cuda') + p.add_argument('--probes', type=int, default=2, help='Reference-checked queries per type') + return parser, add + + +def arguments(parser, args, *, translate=None, skip=()): + """Command-line options that reproduce the non-default values of ``args``.""" + def text(value): + return translate(value) if translate is not None and isinstance(value, Path) else str(value) + + argv = [] + for action in parser._actions: + if not action.option_strings or action.dest in skip: + continue + value = getattr(args, action.dest, action.default) + if value is None or value == action.default: + continue + flag = action.option_strings[-1] + if action.nargs == 0: + argv.append(flag) + elif isinstance(value, list): + argv += [flag, *map(text, value)] + else: + argv += [flag, text(value)] + return argv + + +def resolve(args): + """Recipes selected by ``args``, with the suite's default manifests and answer filter.""" + return prepare_manifest(args.manifests or default_recipes(args.suite), profile=args.profile, + entries=args.entries, methods=args.methods, datasets=args.datasets, + query_types=getattr(args, 'query_types', None), atomic_negation=getattr(args, 'atomic_negation', False), + observed_facts=getattr(args, 'observed_facts', None), inference_graphs=getattr(args, 'inference_graphs', None), + answer_filter=args.answer_filter or SUITES[args.suite]['answer_filter'], suite=args.suite, + hardware_profile=args.hardware_profile, kgfm_batch_size=args.kgfm_batch_size) + + +def ensure_inputs(args, manifest): + # Validation-only evaluation never reads the +H test reduction labels. + test_labels = getattr(args, 'split', 'test') != 'valid' + if not args.no_setup: + inputs.ensure_inputs(manifest, args.input_root, suite=args.suite, models_archive=args.models_archive, + test_labels=test_labels) + else: + inputs.ensure_inputs(manifest, args.input_root, suite=args.suite, download_missing=False, test_labels=test_labels) + + +def in_container(parser, args): + """Run the same command in Docker: inputs read-only at /inputs, OUTPUT writable at /results.""" + root, output = args.input_root.resolve(), args.output.resolve() + if not root.is_dir() or output == root or not output.is_relative_to(root): + raise ValueError('--output must be a directory strictly below an existing --input-root') + + def mapped(value): + path = Path(value).resolve() + if path.is_relative_to(output): + return Path('/results', path.relative_to(output)).as_posix() + if path.is_relative_to(root): + return Path('/inputs', path.relative_to(root)).as_posix() + raise ValueError(f'{value} must be below --input-root') + + argv = [args.suite, args.command, '--input-root', '/inputs', '--output', '/results', + *arguments(parser, args, translate=mapped, skip=HOST_OPTIONS | {'manifest_output_root', 'study_path'})] + if args.command == 'train': + # Generated manifests refer to checkpoints by their path below the input root. + argv += ['--manifest-output-root', Path('/inputs', output.relative_to(root)).as_posix()] + if args.command == 'verify': + argv += ['--study-path', output.relative_to(root).as_posix()] + if any(action.dest == 'no_setup' for action in parser._actions): + argv.append('--no-setup') + if not args.dry_run and not args.no_setup: + manifest = (read_manifest(output / ('verified-bundle' if args.phase == 'test' else 'bundle') / 'bundle.json') + if args.command == 'run' else resolve(args)) + inputs.ensure_inputs(manifest, root, suite=args.suite, models_archive=args.models_archive) + gpu = args.gpu or CONTAINER_GPU[args.command] + if gpu == 'none' and getattr(args, 'device', 'cpu') != 'cpu': + raise ValueError('Without container GPU access (--gpu none), use --device cpu') + if not args.dry_run: + output.mkdir(parents=True, exist_ok=True) + docker.run(args.image, argv, inputs=root, results=output, gpu=gpu, dry_run=args.dry_run) + + +def run_workers(jobs, *, status, gpus=None, workers_per_gpu=1): + """Run (entry, method, command, log) jobs in fresh processes, optionally one GPU per worker. + + Results do not depend on scheduling: each worker has its own checkpoint and + output. After a failure no new entry starts; running workers finish. + Returns the failed (entry, exit code) pairs. + """ + slots = [gpu for gpu in gpus for _ in range(workers_per_gpu)] if gpus else [None] + pending = sorted(jobs, key=lambda job: RUNTIME_ORDER.index(job[1]) if job[1] in RUNTIME_ORDER else len(RUNTIME_ORDER)) + running, failed = {}, [] + + def report(): + write_json(status, dict(state='running', running={entry: dict(pid=process.pid, gpu=gpu) + for process, (entry, gpu, _) in running.items()}, + pending=[job[0] for job in pending], failed=[entry for entry, _ in failed])) + + while running or (pending and not failed): + while pending and slots and not failed: + entry, _, command, log = pending.pop(0) + gpu = slots.pop(0) + print(json.dumps(dict(entry=entry, state='starting', gpu=gpu)), flush=True) + stream = log.open('ab', buffering=0) + environment = os.environ if gpu is None else dict(os.environ, CUDA_VISIBLE_DEVICES=str(gpu)) + process = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=stream, stderr=subprocess.STDOUT, env=environment) + running[process] = entry, gpu, stream + report() + finished = [process for process in running if process.poll() is not None] + if not finished: + time.sleep(0.5) + continue + for process in finished: + entry, gpu, stream = running.pop(process) + stream.close() + slots.append(gpu) + if process.returncode: + failed.append((entry, process.returncode)) + print(json.dumps(dict(entry=entry, state='failed' if process.returncode else 'complete', + exit_code=process.returncode)), flush=True) + report() + return failed + + +def entry_jobs(parser, args, entries, output, *flags): + """One fresh worker process per entry, repeating this command for that entry alone.""" + common = arguments(parser, args, skip={'entries', 'methods', 'datasets', 'gpus', 'workers_per_gpu', 'no_setup', 'dry_run'}) + jobs = [] + for entry in entries: + (output / entry['id']).mkdir(parents=True, exist_ok=True) + jobs.append((entry['id'], entry['method'], + [sys.executable, '-u', '-m', 'benchmarks.cqa', args.suite, args.command, *common, + '--entries', entry['id'], '--no-setup', *flags], + output / entry['id'] / 'worker.log')) + return jobs + + +def finish(jobs, args, output, ids): + failed = run_workers(jobs, status=output / 'status.json', gpus=args.gpus, workers_per_gpu=args.workers_per_gpu) + if failed: + write_json(output / 'status.json', dict(state='failed', exit_codes=dict(failed))) + raise SystemExit('Failed entries (see worker.log): ' + ', '.join(entry for entry, _ in failed)) + write_json(output / 'status.json', dict(state='complete', entries=ids)) + + +def setup(parser, args): + manifest = resolve(args) + if args.dry_run: + inputs.describe(manifest, args.input_root.resolve(), suite=args.suite) + elif args.check: + if inputs.check(manifest, args.input_root.resolve(), suite=args.suite): + raise SystemExit(1) + else: + inputs.ensure_inputs(manifest, args.input_root, suite=args.suite, models_archive=args.models_archive) + print('Inputs ready. No model was loaded and no benchmark was started.') + + +def evaluate(parser, args): + manifest = resolve(args) + if args.dry_run: + print(json.dumps(manifest, indent=2)) + return + ensure_inputs(args, manifest) + if args.gpus: + finish(entry_jobs(parser, args, manifest['entries'], args.output), args, args.output, + [entry['id'] for entry in manifest['entries']]) + return + from .evaluate import evaluate_manifest + evaluate_manifest(manifest, input_root=args.input_root, output=args.output, device=args.device, split=args.split, + limit=args.max_queries_per_shape, seed=args.seed, threads=args.threads) + + +def prepare(parser, args): + if (args.output / 'bundle' / 'bundle.json').exists(): + raise ValueError('Use a new output directory to change a frozen profile') + manifest = resolve(args) + if args.dry_run: + print(json.dumps(manifest, indent=2)) + return + ensure_inputs(args, manifest) + from .study import freeze + bundle = freeze(manifest, args.output / 'bundle', args.input_root) + write_json(args.output / 'manifest.json', manifest) + print(json.dumps(dict(profile=args.profile, entries=len(manifest['entries']), source=bundle['source_sha256']))) + + +def verify(parser, args): + """Run every entry's parity check in a fresh process, then freeze the passing evidence. + + Passing evidence of an unchanged entry (a KGFM integration check, or a + parity check against the same oracles) is reused, so an interrupted + verification resumes and checks can run as separate jobs. With ``--gpus`` + the checks run concurrently, one GPU per worker, logged to + ``verification/ENTRY.log``; the evidence does not depend on the order. + """ + from .study import freeze, integration_passed, parity_passed, verify_bundle + study, root = args.output.resolve(), args.input_root.resolve() + if (study / 'verified-bundle' / 'bundle.json').exists(): + raise ValueError('A verified bundle already exists; use a new output directory for new evidence') + # Evidence is pinned by its path below the input root, valid on the host and in containers. + if args.study_path is None and not study.is_relative_to(root): + raise ValueError('The study directory must be below --input-root, where evidence paths are recorded') + relative = Path(args.study_path) if args.study_path is not None else study.relative_to(root) + bundle = verify_bundle(study / 'bundle', root) + manifest = bundle['manifest'] + jobs = [] + for entry in manifest['entries']: + common = [sys.executable, '-u', '-m', 'benchmarks.cqa', args.suite] + options = ['--bundle', str(study / 'bundle'), '--input-root', str(root), '--entry', entry['id'], '--device', args.device] + if entry['method'].endswith('-adapter'): + target = study / 'verification' / entry['id'] + if integration_passed(target / 'integration.json', bundle, entry): + continue + if (target / 'pilot').exists(): + raise ValueError(f'KGFM verification requires a fresh directory: {target}') + jobs.append((entry['id'], entry['method'], [*common, 'integration', *options, '--output', str(target)])) + continue + reference = args.references / f'{entry["id"]}.pt' if args.references else None + if reference is None or not reference.is_file(): + raise ValueError(f'Missing baseline oracle {entry["id"]}.pt; pass --references') + evidence = study / 'evidence' / f'{entry["id"]}.json' + command = [*common, 'parity', *options, '--reference', str(reference), '--output', str(evidence)] + comparison = None + if args.comparison_references and entry['method'] in ('cqd', 'cqd-hybrid'): + comparison = args.comparison_references / f'{entry["id"]}.pt' + if not comparison.is_file(): + raise ValueError(f'Missing comparison oracle: {comparison}') + command += ['--comparison-reference', str(comparison)] + if not parity_passed(evidence, bundle, entry, reference, comparison): + jobs.append((entry['id'], entry['method'], command)) + if args.gpus and jobs: + logs = study / 'verification' + logs.mkdir(parents=True, exist_ok=True) + failed = run_workers([(entry, method, command, logs / f'{entry}.log') for entry, method, command in jobs], + status=logs / 'status.json', gpus=args.gpus, workers_per_gpu=args.workers_per_gpu) + if failed: + raise SystemExit('Verification failed (see verification/ENTRY.log): ' + ', '.join(entry for entry, _ in failed)) + else: + for entry, _, command in jobs: + print(json.dumps(dict(entry=entry, state='verifying')), flush=True) + if subprocess.run(command).returncode: + raise SystemExit(f'Verification of {entry} failed; see its output above') + for entry in manifest['entries']: + evidence = study / 'evidence' / f'{entry["id"]}.json' + if entry['method'].endswith('-adapter'): + evidence.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(study / 'verification' / entry['id'] / 'integration.json', evidence) + if not json.loads(evidence.read_text()).get('passed'): + raise ValueError(f'Verification failed: {entry["id"]}') + entry['verification'] = (relative / 'evidence' / evidence.name).as_posix() + write_json(study / 'verified-manifest.json', manifest) + freeze(manifest, study / 'verified-bundle', root) + print(json.dumps(dict(verified=len(manifest['entries']), bundle=str(study / 'verified-bundle')))) + + +def run(parser, args): + from dicee.query_answering._checkpoint import BenchmarkCheckpoint, checksum + + from .study import run_job, verify_bundle + bundle_dir = args.output / ('verified-bundle' if args.phase == 'test' else 'bundle') + results = args.output / args.phase + if not args.worker: + ensure_inputs(args, read_manifest(bundle_dir / 'bundle.json')) + bundle = verify_bundle(bundle_dir, args.input_root) + entries = select_entries(bundle['manifest']['entries'], entry_ids=args.entries, methods=args.methods, datasets=args.datasets) + if args.worker: + if len(entries) != 1: + raise ValueError('A worker takes one entry') + result = run_job(bundle_dir, entries[0]['id'], args.input_root, results / entries[0]['id'], + device=args.device, phase=args.phase, pilot_queries=args.pilot_queries) + print(json.dumps(dict(entry=entries[0]['id'], queries=result['queries'], averages=result['averages'])), flush=True) + return + # Parallelism is excluded from the suite identity: it cannot change results. + identity = dict(bundle=checksum(bundle_dir / 'bundle.json'), phase=args.phase, device=args.device, pilot_queries=args.pilot_queries) + with BenchmarkCheckpoint(results / 'suite', identity): + finish(entry_jobs(parser, args, entries, results, '--worker'), args, results, [entry['id'] for entry in entries]) + + +def report(parser, args): + from .reports import export_reports + published = args.published or suite_directory(args.suite) / 'published_results.json' + rows = export_reports(args.results, published if published.is_file() else None, args.output, + bootstrap_samples=args.bootstrap_samples, seed=args.seed, title=SUITES[args.suite]['title'], + workers=args.workers) + print(json.dumps(dict(results=len(rows), output=str(args.output)))) + + +def difficulty_report(parser, args): + from .difficulty import export_difficulty_reports + result = export_difficulty_reports(args.results, args.bundle, args.input_root, args.output, + labels_root=args.labels_root, entries=args.entries, query_types=args.query_types, + direct=args.direct) + print(json.dumps(dict(rows=len(result['rows']), pending_main_entries=len(result['pending_main_entries']), output=str(args.output)))) + + +def train(parser, args): + if args.suite != 'ultraquery': + raise ValueError('Baseline training belongs to the UltraQuery suite') + from .train import run as run_training + run_training(args) + + +def build(parser, args): + docker.build(args.image, cache=args.cache, network=args.network, training_base=args.training) + + +def parity(parser, args): + from .oracles import verify_predictions + result = verify_predictions(args.bundle, args.entry, args.input_root, args.reference, args.output, + device=args.device, comparison_reference=args.comparison_reference) + print(json.dumps(dict(passed=result['passed'], queries=len(result['queries'])))) + if not result['passed']: + raise SystemExit(1) + + +def integration(parser, args): + from .verification.verify_kgfm import verify as verify_kgfm + verify_kgfm(args.bundle, args.entry, args.input_root, args.output, device=args.device, probes=args.probes) + + +HANDLERS = {'setup': setup, 'evaluate': evaluate, 'prepare': prepare, 'verify': verify, 'run': run, 'report': report, + 'difficulty-report': difficulty_report, 'train': train, 'build': build, 'parity': parity, + 'integration': integration} + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else list(argv) + parser, commands = build_parser() + args = parser.parse_args(argv) + command = commands[args.command] + if getattr(args, 'gpus', None) is not None or getattr(args, 'workers_per_gpu', 1) != 1: + if args.workers_per_gpu < 1 or not args.gpus or len(set(args.gpus)) != len(args.gpus): + command.error('--gpus needs distinct devices and --workers-per-gpu a positive count') + if args.device not in ('cuda', 'cuda:0'): + command.error('With --gpus, use --device cuda; each worker sees only its assigned GPU') + for name in ('kgfm_batch_size', 'max_queries_per_shape', 'pilot_queries', 'bootstrap_samples', 'probes', 'workers'): + if getattr(args, name, None) is not None and getattr(args, name) < (0 if name == 'bootstrap_samples' else 1): + command.error(f'--{name.replace("_", "-")} must be positive') + try: + if getattr(args, 'image', None) and args.command in CONTAINER_GPU: + return in_container(command, args) + return HANDLERS[args.command](command, args) + except ValueError as error: + command.error(str(error)) diff --git a/benchmarks/cqa/difficulty.py b/benchmarks/cqa/difficulty.py new file mode 100644 index 000000000..da70eab5b --- /dev/null +++ b/benchmarks/cqa/difficulty.py @@ -0,0 +1,481 @@ +"""CPU-only answer-level difficulty analysis of immutable +H rank traces. + +Truth edges are label-generation input, never inputs to a predictor. Released +reduction names and minimum missing-positive-edge counts are distinct analyses: +the authors' test reduction files use train+valid (despite the paper's shorthand +"train"); in particular their union reductions are not min-plus edge counts. +""" + +import csv +import json +import math +import pickle +import sqlite3 +from collections import Counter, OrderedDict, defaultdict +from pathlib import Path + +import numpy as np + +import dicee.query_answering +from dicee.query_answering._checkpoint import checksum, read, write_json +from dicee.query_answering._query import PLUS_H_SHAPES, QUERY_SHAPES, compile_query, exact_answers +from dicee.query_answering.benchmark import METRICS +from dicee.query_answering.datasets import PLUS_H_REFERENCE_COMMIT, BenchmarkQuery, dataset_spec + +# This postprocessor and the library code that loads queries and computes metrics. +POSTPROCESSOR = (Path(__file__), *(Path(dicee.query_answering.__file__).with_name(name) for name in ( + '_query.py', 'context.py', '_checkpoint.py', 'datasets.py', 'benchmark.py', 'method_evaluation.py'))) + + +def postprocessor_files(): + return {f'{path.parent.name}/{path.name}': checksum(path) for path in POSTPROCESSOR} + + +# Explicit allowlist from the pinned author release; never treat all/ as a bin. +REDUCTIONS = { + '2p': ('1p', '2p'), '3p': ('1p', '2p', '3p'), '4p': ('1p', '2p', '3p', '4p'), + '2i': ('1p', '2i'), '3i': ('1p', '2i', '3i'), '4i': ('1p', '2i', '3i', '4i'), + 'ip': ('1p', '2i', '2p', 'ip'), 'pi': ('1p', '2i', '2p', 'pi'), + 'up': ('1p', '2u', 'up'), + **{s: ('pos-exist', 'pos-only-miss') for s in ('3in', 'pin', 'inp')}, +} + + +class DifficultyClassifier: + """Min-plus evaluation on valid full-truth witnesses of anchored trees. + + Cache size is bounded by the total number of entity records, not just keys. + Negative branches restrict truth and contribute no positive-edge cost. + """ + + def __init__(self, full_outgoing, observed_outgoing, num_entities, *, cache_answers=100000): + self.full = full_outgoing + self.observed = observed_outgoing + self.num_entities = num_entities + self.cache_answers = cache_answers + self.cache = OrderedDict() + self.cached_answers = 0 + + def costs(self, node): + if node in self.cache: + self.cache.move_to_end(node) + return self.cache[node] + op = node[0] + if op == 'anchor': + result = {node[1]: 0} + elif op == 'project': + result = {} + for head, cost in self.costs(node[2]).items(): + seen = self.observed.get(head, {}).get(node[1], ()) + for tail in self.full.get(head, {}).get(node[1], ()): + value = cost + int(tail not in seen) + if tail not in result or value < result[tail]: + result[tail] = value + elif op == 'and': + positives = [self.costs(c) for c in node[1:] if c[0] != 'not'] + if not positives: + raise ValueError('Difficulty requires a positive anchored branch') + smallest = min(positives, key=len) + result = {a: sum(part[a] for part in positives) for a in smallest if all(a in part for part in positives)} + for child in node[1:]: + if child[0] == 'not': + # Includes negated paths (pni), and restricts intermediate + # witnesses before the final projection in inp. + for answer in exact_answers(child[1], self.full, self.num_entities): + result.pop(answer, None) + elif op == 'or': + result = {} + for child in node[1:]: + for answer, value in self.costs(child).items(): + result[answer] = min(result.get(answer, value), value) + else: + raise ValueError(f'Unsupported difficulty operator: {op}') + size = max(1, len(result)) + if size <= self.cache_answers: + while self.cache and self.cached_answers + size > self.cache_answers: + _, evicted = self.cache.popitem(last=False) + self.cached_answers -= max(1, len(evicted)) + self.cache[node] = result + self.cached_answers += size + return result + + def classify(self, shape, query, hard): + if shape not in PLUS_H_SHAPES: + raise ValueError(f'Unsupported difficulty shape: {shape}') + tree = compile_query(query) + maximum = positive_edges(tree) + costs = self.costs(tree) + if set(hard) - costs.keys(): + raise ValueError(f'Hard targets are not true on the full graph: {shape} {query}') + result = {a: costs[a] for a in hard} + if any(not 0 <= cost <= maximum for cost in result.values()): + raise ValueError('Missing-positive-edge count outside tree bounds') + return result, maximum + + +def positive_edges(node): + if node[0] in ('anchor', 'not'): + return 0 + if node[0] == 'project': + return 1 + positive_edges(node[2]) + counts = [positive_edges(child) for child in node[1:]] + if node[0] == 'or': + if len(set(counts)) != 1: + raise ValueError('Unequal-size union branches have no single full-inference threshold') + return counts[0] + if node[0] == 'and': + return sum(counts) + raise ValueError('Unsupported difficulty tree') + + +def difficulty_name(cost, maximum): + return 'observed' if cost == 0 else 'full' if cost == maximum else 'partial' + + +def load_graphs(folder): + """Read public-ID truth/observed adjacencies once per dataset, CPU only.""" + folder = Path(folder) + graph = folder / 'KG_splits' if (folder / 'KG_splits').exists() else folder + full, train, extended = (defaultdict(lambda: defaultdict(set)) for _ in range(3)) + files = {} + for split in ('train', 'valid', 'test'): + path = graph / f'{split}.txt' + files[path.relative_to(folder).as_posix()] = checksum(path) + with path.open() as stream: + for line in stream: + if not line.strip(): + continue + h, r, t = map(int, line.split()) + # Same public reciprocal pairing as load_benchmark. + for head, relation, tail in ((h, r, t), (t, r ^ 1, h)): + full[head][relation].add(tail) + if split == 'train': + train[head][relation].add(tail) + if split != 'test': + extended[head][relation].add(tail) + with (folder / 'id2ent.pkl').open('rb') as stream: + n = len(pickle.load(stream)) + return full, {'train': train, 'train+valid': extended}, n, files + + +def root_queries(folder, shapes): + """Recover trace identity from root labels, including original easy filters. + + Corrected filters do not change RankTrace's identity or its hard targets. + Loading the raw pickles separately avoids retaining their many unused keys. + """ + folder = Path(folder) + with (folder / 'test-queries.pkl').open('rb') as stream: + structured = pickle.load(stream) + selected = {q: shape for shape in shapes for q in structured[QUERY_SHAPES[shape]]} + values = {} + for kind in ('easy', 'hard'): + with (folder / f'test-{kind}-answers.pkl').open('rb') as stream: + raw = pickle.load(stream) + if selected.keys() - raw.keys(): + raise ValueError(f'Missing root {kind} answers') + values[kind] = {q: frozenset(raw[q]) for q in selected} + del raw + records = {} + for q, shape in selected.items(): + easy, hard = values['easy'][q], values['hard'][q] + if not hard or easy & hard: + raise ValueError('Root hard answers must be nonempty and disjoint from easy answers') + key = BenchmarkQuery(shape, q, easy, hard).identity + records[key] = dict(shape=shape, query=q, hard=hard) + if len(records) != len(selected): + raise ValueError('Duplicate root query identities') + return records + + +def released_labels(folder, records, *, dataset, required=False): + """Join released hard-answer partitions to the retained root hard subset. + + A wholly absent family is explicitly unavailable unless required. A present + family must be complete, disjoint and cover every retained hard answer. + Extra released answers are counted and excluded, never added to evaluation. + """ + folder = Path(folder) + by_shape = defaultdict(dict) + for key, record in records.items(): + by_shape[record['shape']][record['query']] = (key, record['hard']) + labels, provenance = {}, {} + for shape, selected in by_shape.items(): + bins = REDUCTIONS.get(shape) + if bins is None or dataset == 'ICEWS18+H': + provenance[shape] = dict(status='unavailable', reason='No released reduction partition for this dataset/type') + continue + paths = {label: folder / 'test-query-reduction' / shape / label / 'test-hard-answers.pkl' for label in bins} + present = [p.exists() for p in paths.values()] + if not any(present) and not required: + provenance[shape] = dict(status='unavailable', reason='Released reduction files not supplied') + continue + if not all(present): + raise FileNotFoundError(f'Missing released reduction files: {[str(p) for p in paths.values() if not p.exists()]}') + assignments = {q: {} for q in selected} + extra = 0 + for label, path in paths.items(): + with path.open('rb') as stream: + raw = pickle.load(stream) + for q, answers in raw.items(): + if q not in selected: + extra += len(answers) + continue + _, hard = selected[q] + extra += len(set(answers) - hard) + for answer in set(answers) & hard: + if answer in assignments[q]: + raise ValueError(f'Overlapping released reduction labels: {shape} {q} {answer}') + assignments[q][answer] = label + for q, (key, hard) in selected.items(): + if assignments[q].keys() != hard: + raise ValueError(f'Missing released reduction labels for root hard targets: {shape} {q}') + labels[key] = assignments[q] + provenance[shape] = dict(status='available', label_reference_graph='train+valid', excluded_extra_answers=extra, + files={str(p): checksum(p) for p in paths.values()}) + return labels, provenance + + +def summarize_difficulty(path, records, classifier, *, author_labels=None, shapes=None): + """Conditional answer→query means plus additive original-query contributions.""" + totals = defaultdict(lambda: dict(queries=0, hard_answers=0, sums=np.zeros((2, 4)), + contribution=np.zeros((2, 4)), query_mass=0.)) + counts = defaultdict(Counter) + seen = set() + harmonic = np.zeros(1) + with sqlite3.connect(f'{Path(path).resolve().as_uri()}?mode=ro', uri=True) as connection: + for key, shape, payload in connection.execute('SELECT query_id, shape, answers FROM queries ORDER BY position'): + if shapes is not None and shape not in shapes: + continue + if key in seen or key not in records or records[key]['shape'] != shape: + raise ValueError('Trace query identity/shape does not match root labels') + seen.add(key) + record = records[key] + rows = json.loads(payload) + if not rows or any(len(row) != 4 or any(type(x) is not int for x in row) for row in rows): + raise ValueError('Malformed answer-rank records') + arr = np.asarray(rows, dtype=np.int64) + answers, ranks, greater, sizes = arr.T + if len(set(answers)) != len(answers) or set(answers) != record['hard']: + raise ValueError('Trace answer identities differ from the exact root hard targets') + if (greater < 0).any() or (sizes < 1).any() or (ranks <= greater).any() or (ranks > greater + sizes).any(): + raise ValueError('Inconsistent filtered rank / tie block') + last = greater + sizes + if last.max() >= len(harmonic): + harmonic = np.concatenate(([0.], np.cumsum(1. / np.arange(1, last.max() + 1)))) + values = np.stack((np.stack((1. / ranks, *(ranks <= k for k in (1, 3, 10))), axis=1), + np.stack(((harmonic[last] - harmonic[greater]) / sizes, + *(np.clip(k - greater, 0, sizes) / sizes for k in (1, 3, 10))), axis=1))) + costs, maximum = classifier.classify(shape, record['query'], record['hard']) + groups = defaultdict(list) + released = (author_labels or {}).get(key) + for index, answer in enumerate(answers): + cost = costs[answer] + groups['difficulty', difficulty_name(cost, maximum)].append(index) + groups['inferred_positive_edges', str(cost)].append(index) + groups['released_reduction', released[answer] if released else 'unavailable'].append(index) + groups['overall', 'all'] = list(range(len(answers))) + counts[shape].update(queries=1, hard_answers=len(answers), + labeled_author_answers=len(answers) if released else 0) + for (level, label), indices in groups.items(): + row = totals[shape, level, label, maximum] + selected = values[:, indices, :] + row['queries'] += 1 + row['hard_answers'] += len(indices) + row['sums'] += selected.mean(axis=1) + row['contribution'] += selected.sum(axis=1) / len(answers) + row['query_mass'] += len(indices) / len(answers) + if not seen: + raise ValueError('No completed trace queries for the requested types') + result = [] + for (shape, level, label, maximum), values in sorted(totals.items()): + count = counts[shape]['queries'] + result.append(dict(shape=shape, grouping=level, label=label, positive_edges=maximum, + one_positive_edge=maximum == 1, queries=values['queries'], hard_answers=values['hard_answers'], + parent_queries=count, parent_hard_answers=counts[shape]['hard_answers'], + query_weight=values['query_mass'] / count, + **{policy: dict(zip(METRICS, values['sums'][i] / values['queries'])) for i, policy in enumerate(('sort', 'expected'))}, + original_weight_contribution={policy: dict(zip(METRICS, values['contribution'][i] / count)) + for i, policy in enumerate(('sort', 'expected'))})) + return dict(rows=result, counts=dict(counts), seen=seen) + + +def export_difficulty_reports(results, bundle, input_root, output, *, labels_root=None, entries=None, query_types=None, + direct=False): + """Postprocess completed jobs from a frozen inference bundle. + + No frozen-source verification, model loading, checkpoint locks or writes to + inference artifacts. A new output directory is required. In-progress traces + without result.json are never opened. + + ``direct`` reads the results of direct evaluate runs, which carry no bundle + hash. The bundle then only pins the label-generation inputs and the filter + corrections. + """ + from dicee.query_answering.method_evaluation import GRAPH_INDEPENDENT_METHODS + + results, bundle, input_root, output = map(lambda p: Path(p).resolve(), (results, bundle, input_root, output)) + if output.exists() or any(output.is_relative_to(p) for p in (results, bundle)): + raise ValueError('Use a new report directory outside the frozen bundle and inference results') + source_files = postprocessor_files() + frozen = read(bundle / 'bundle.json') + bundle_hash = checksum(bundle / 'bundle.json') + shapes = set(PLUS_H_SHAPES if query_types is None else query_types) + if not shapes or shapes - set(PLUS_H_SHAPES): + raise ValueError('Select supported +H query types') + selected_entries = set(entries) if entries is not None else None + jobs = [] + completed = set() + for path in sorted(results.rglob('result.json')): + report = read(path) + run = report.get('benchmark_run', report.get('paper_run')) + if run is None: + continue + matches = (run.get('mode') == 'direct' and 'bundle' not in run) if direct else run.get('bundle') == bundle_hash + if matches: + completed.add(run['entry']) + if selected_entries is not None and run['entry'] not in selected_entries: + continue + if not matches: + raise ValueError(f'Result was not produced by {"a direct run" if direct else "this frozen bundle"}: {path}') + if report['split'] != 'test': + raise ValueError('Inference difficulty currently supports test traces only') + jobs.append((path, report, run)) + if not jobs: + raise ValueError('No completed matching benchmark results') + if selected_entries is not None and selected_entries - completed: + raise ValueError(f'Requested entries are not completed: {sorted(selected_entries - completed)}') + expected_jobs = {entry['id'] for entry in frozen['manifest']['entries']} + dataset_jobs = defaultdict(list) + for job in jobs: + dataset_jobs[job[1]['dataset']].append(job) + all_rows, details, sources = [], [], {} + for dataset, jobs in sorted(dataset_jobs.items()): + folder = input_root / frozen['manifest']['data_root'] / dataset_spec(dataset)[1] + available_shapes = shapes & {s for _, r, _ in jobs for s in r['per_shape']} + if not available_shapes: + continue + # Check consumed root labels and all label-generation graph files against + # the bundle's pins. This intentionally does not compare current code or + # load/checksum frozen model weights. + graph_prefix = 'KG_splits/' if dataset == 'ICEWS18+H' else '' + names = ['test-queries.pkl', 'test-easy-answers.pkl', 'test-hard-answers.pkl', 'id2ent.pkl', + *(graph_prefix + s + '.txt' for s in ('train', 'valid', 'test'))] + pins = {} + for name in names: + path = folder / name + key = str(path.relative_to(input_root)) + digest = checksum(path) + if frozen['files'].get(key) != digest: + raise ValueError(f'Consumed label-generation input is not pinned by bundle: {path}') + pins[name] = digest + records = root_queries(folder, available_shapes) + label_folder = (Path(labels_root).resolve() / dataset_spec(dataset)[1]) if labels_root else folder + author, author_source = released_labels(label_folder, records, dataset=dataset, required=labels_root is not None) + full, observed, n, graph_files = load_graphs(folder) + graph = 'train+valid' + classifier = DifficultyClassifier(full, observed[graph], n) + # Retain only classifications of root hard targets (not all reachable + # entities) across model/filter/adapter rows, bounded by this dataset. + target_labels = {} + + class SharedClassifier: + def __init__(self, classifier, cache): + self.classifier = classifier + self.cache = cache + + def classify(self, shape, query, hard): + key = (shape, query) + if key not in self.cache: + self.cache[key] = self.classifier.classify(shape, query, hard) + return self.cache[key] + + sources[dataset] = dict(root_files=pins, truth_graph_files=graph_files, released=author_source) + available = Counter(record['shape'] for record in records.values()) + for path, report, run in jobs: + if not (available_shapes & report['per_shape'].keys()): + continue + trace = path.parent / 'ranks.sqlite3' + if checksum(trace) != report['rank_trace_sha256']: + raise ValueError(f'Rank trace checksum mismatch: {trace}') + filter_policy = report['protocol'].get('answer_filter', 'released') + if filter_policy == 'corrected': + correction = frozen.get('filter_corrections', {}).get(dataset) + if not correction or checksum(bundle / correction['path']) != correction['sha256']: + raise ValueError('Corrected-filter result lacks a pinned correction audit') + method = report['inference']['method'] + inference_graph = report['dataset_metadata']['inference_graph'] + summary = summarize_difficulty(trace, records, SharedClassifier(classifier, target_labels), + author_labels=author, shapes=available_shapes) + for row in summary['rows']: + shape = row['shape'] + if row['grouping'] == 'overall': + online_counts = report['per_shape'][shape] + if (row['queries'] != online_counts['queries'] or row['hard_answers'] != online_counts['hard_answers'] + or (report['protocol']['full_split'] and row['queries'] != available[shape])): + raise ValueError('Completed trace coverage differs from evaluation/root query coverage') + for policy in ('sort', 'expected'): + online = (report['per_shape'][shape] if policy == 'sort' else + report['additional_tie_metrics'][policy]['per_shape'][shape]) + if any(not math.isclose(row[policy][m], online[m], abs_tol=1e-11) for m in METRICS): + raise ValueError(f'Offline rank metrics differ from completed evaluation: {path}') + row.update(entry=run['entry'], dataset=dataset, method=method, + calibration=report['inference'].get('calibration'), answer_filter=filter_policy, + inference_graph=inference_graph, comparison_graph=graph, + graph_independent=method in GRAPH_INDEPENDENT_METHODS, + label_reference_graph=graph, label_graph_matches_comparison=True, + available_queries=available[shape], + complete_shape=row['parent_queries'] == available[shape]) + all_rows.append(row) + details.append(dict(entry=run['entry'], dataset=dataset, comparison_graph=graph, + result=str(path), result_sha256=checksum(path), rank_trace_sha256=report['rank_trace_sha256'], + coverage=summary['counts'], available_queries=dict(available), + full_selected_types=all(c['queries'] == available[s] for s, c in summary['counts'].items()), + complete_benchmark_types=set(summary['counts']) == set(PLUS_H_SHAPES), + protocol_full_split=report['protocol']['full_split'])) + del records, full, observed, classifier, target_labels, author + if not all_rows: + raise ValueError('No completed results contain selected query types') + if postprocessor_files() != source_files: + raise ValueError('Postprocessor source changed during reporting; rerun with stable source') + payload = dict(version=1, bundle_sha256=bundle_hash, frozen_inference_source_sha256=frozen['source_sha256'], + postprocessor_files=source_files, + author_commit=PLUS_H_REFERENCE_COMMIT, author_release='benchs-1.0', + author_label_reference_graph='train+valid', + hardness_reference_graph='train+valid', + semantics='Minimum missing positive atoms on full-truth witnesses; negation restricts truth at zero cost; union selects a branch.', + averaging='Within each group: answers per query, then participating queries per parent type. No cross-type difficulty macro.', + contribution='original_weight_contribution sums to the parent metric within each grouping; query_weight is its original answer-per-query mass.', + pending_main_entries=[] if direct else sorted(expected_jobs - completed), selected_types=sorted(shapes), + result_mode='direct' if direct else 'bundle', + sources=sources, details=details, rows=all_rows) + write_json(output / 'inference-difficulty.json', payload) + fields = ['entry', 'dataset', 'method', 'calibration', 'answer_filter', 'inference_graph', 'comparison_graph', + 'graph_independent', 'label_reference_graph', 'label_graph_matches_comparison', 'shape', 'grouping', 'label', + 'positive_edges', 'one_positive_edge', 'queries', 'hard_answers', 'parent_queries', 'parent_hard_answers', + 'available_queries', 'complete_shape', 'query_weight'] + with (output / 'inference-difficulty.csv').open('w', newline='') as stream: + metric_fields = [f'{p}_{m}' for p in ('sort', 'expected') for m in METRICS] + writer = csv.DictWriter(stream, fieldnames=fields + metric_fields) + writer.writeheader() + for row in all_rows: + writer.writerow({k: row[k] for k in fields} | {f'{p}_{m}': row[p][m] for p in ('sort', 'expected') for m in METRICS}) + lines = ['# Answer-level inference difficulty', '', + 'Metrics in the CSV/JSON are fractions. The table reports MRR ×100. Each row retains its original query type.', '', + 'All hardness categories use train+valid facts as their fixed reference, independently of the predictor\'s inference graph. ' + 'Full means all positive atoms in a chosen proof are missing. One-positive-edge types ' + '(including 1p and 2u) are identified explicitly and are not evidence of multi-hop reasoning.', '', + 'Only released hard targets are scored. Negated branches constrain full-graph truth but add no positive-edge cost. ' + 'Author union reductions need not equal minimum-edge groups.', '', + 'Group scores average answers within each participating query, then queries. JSON includes additive contributions ' + 'under the original query weights. No difficulty macro averages mix parent types.', '', + f'Pending main jobs: {len(payload["pending_main_entries"])}. Only completed result files were read.', '', + '| Entry | Filters | Type | Difficulty | Positive atoms | Queries | Answers | Sort | Expected |', + '|---|---|---|---|---:|---:|---:|---:|---:|'] + for row in all_rows: + if row['grouping'] == 'difficulty': + lines.append(f'| {row["entry"]} | {row["answer_filter"]} | {row["shape"]} | {row["label"]} | ' + f'{row["positive_edges"]} | {row["queries"]} | {row["hard_answers"]} | ' + f'{100 * row["sort"]["mrr"]:.2f} | {100 * row["expected"]["mrr"]:.2f} |') + (output / 'inference-difficulty.md').write_text('\n'.join(lines) + '\n') + return payload diff --git a/benchmarks/cqa/docker.py b/benchmarks/cqa/docker.py new file mode 100644 index 000000000..4f479983d --- /dev/null +++ b/benchmarks/cqa/docker.py @@ -0,0 +1,91 @@ +"""Build the benchmark image and run commands in it with immutable inputs. + +Containers have no network, mount the input root read-only at ``/inputs`` and +one results directory at ``/results``, and record the image ID they used. +""" + +import json +import os +import shlex +import shutil +import subprocess +import tempfile +from pathlib import Path + +from .manifests import REPO + +HERE = Path(__file__).parent +# Tracked sources copied into the image: the library, this harness, and suite data. +SOURCES = ('dicee', 'pyproject.toml', 'benchmarks/__init__.py', 'benchmarks/cqa', 'benchmarks/adapters', + 'benchmarks/plus_h', 'benchmarks/ultraquery') + + +def inspect(image): + return json.loads(subprocess.check_output(['docker', 'image', 'inspect', image], text=True))[0] + + +def build(image, *, cache=None, network='default', training_base=None): + """Build the pinned runtime, or the CPU author-training runtime on top of ``training_base``.""" + if training_base is not None: + subprocess.run(['docker', 'build', '--network', network, '-t', image, + '--build-arg', f'BENCHMARK_IMAGE={inspect(training_base)["Id"]}', str(HERE / 'training')], check=True) + return + + def git(*arguments): + return subprocess.check_output(['git', '-C', str(REPO), *arguments], text=True) + + commit = git('rev-parse', 'HEAD').strip() + dirty = bool(git('status', '--porcelain', '--untracked-files=no', '--', *SOURCES).strip()) + cache = Path(cache or Path.home() / '.cache' / 'dicee' / 'cqa').resolve() + wheels, pip_cache = cache / 'wheels', Path.home() / '.cache' / 'pip' + wheels.mkdir(parents=True, exist_ok=True) + pip_cache.mkdir(parents=True, exist_ok=True) + base = (HERE / 'Dockerfile').read_text().splitlines()[0].split()[1] + # Resolve hash-pinned wheels once, in the image's own Python, then build offline. + subprocess.run(['docker', 'run', '--rm', '--network', network, '--user', f'{os.getuid()}:{os.getgid()}', + '--env', 'PIP_CACHE_DIR=/pip-cache', + '--mount', f'type=bind,source={wheels},target=/wheels', + '--mount', f'type=bind,source={pip_cache},target=/pip-cache', + '--mount', f'type=bind,source={HERE / "requirements.lock"},target=/requirements.lock,readonly', + base, 'python', '-m', 'pip', 'download', '--require-hashes', '--only-binary=:all:', + '--timeout', '180', '--retries', '10', '--dest', '/wheels', '-r', '/requirements.lock'], check=True) + with tempfile.TemporaryDirectory(prefix='build-', dir=cache) as directory: + shutil.copytree(wheels, Path(directory) / 'wheels', copy_function=os.link) + for relative in git('ls-files', '-z', '--', *SOURCES).split('\0'): + source, target = REPO / relative, Path(directory) / relative + if relative and source.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + subprocess.run(['docker', 'build', '--network', network, '-t', image, '-f', 'benchmarks/cqa/Dockerfile', + '--build-arg', f'SOURCE_COMMIT={commit}', '--build-arg', f'SOURCE_DIRTY={str(dirty).lower()}', + directory], check=True) + + +def run(image, arguments, *, inputs, results, gpu='runtime', dry_run=False): + """Run ``python -m benchmarks.cqa ARGUMENTS`` in ``image``. + + Every container of a results directory must use the same image ID; the + first launch records it in ``container-image.json``. + """ + inputs, results = Path(inputs).resolve(), Path(results).resolve() + mounts = ['--mount', f'type=bind,source={inputs},target=/inputs,readonly', + '--mount', f'type=bind,source={results},target=/results', + '--mount', f'type=bind,source={results / "cache"},target=/cache'] + devices = {'runtime': ['--gpus', 'all'], 'cdi': ['--device', 'nvidia.com/gpu=all'], 'none': []}[gpu] + command = ['docker', 'run', '--rm', '--network', 'none', '--read-only', '--shm-size', '2g', + '--tmpfs', '/tmp:rw,exec,nosuid,size=4g', '--user', f'{os.getuid()}:{os.getgid()}', + '--env', 'HOME=/tmp', '--env', 'XDG_CACHE_HOME=/cache', '--env', 'TORCHINDUCTOR_CACHE_DIR=/cache/inductor', + '--env', 'TRITON_CACHE_DIR=/cache/triton', *mounts, *devices] + if dry_run: + print(shlex.join([*command, image, *arguments])) + return + identity = inspect(image) + pin = results / 'container-image.json' + if pin.exists() and json.loads(pin.read_text())['image_id'] != identity['Id']: + raise ValueError('Image differs from this study; use the original image ID or a new output directory') + (results / 'cache').mkdir(parents=True, exist_ok=True) + if not pin.exists(): + pin.write_text(json.dumps(dict(image_id=identity['Id'], repo_digests=identity.get('RepoDigests', [])), indent=2) + '\n') + command += ['--env', f'DICEE_IMAGE_ID={identity["Id"]}', identity['Id'], *arguments] + (results / 'container-launch.json').write_text(json.dumps(dict(image_id=identity['Id'], command=command), indent=2) + '\n') + subprocess.run(command, check=True) diff --git a/benchmarks/cqa/evaluate.py b/benchmarks/cqa/evaluate.py new file mode 100644 index 000000000..39ba7ab7d --- /dev/null +++ b/benchmarks/cqa/evaluate.py @@ -0,0 +1,44 @@ +"""Evaluate selected entries of a portable study manifest directly, without a frozen bundle.""" + +import json +from pathlib import Path + +from dicee.query_answering._checkpoint import checksum, input_path +from dicee.query_answering.method_evaluation import evaluate_method +from dicee.query_answering.methods import REFERENCES + +from .provenance import validate_training + + +def evaluate_manifest(manifest, *, input_root, output, device='cpu', split='test', limit=None, seed=None, threads=None): + """Evaluate each entry into ``output/ID``, resolving its inputs below ``input_root``.""" + seed = manifest.get('seed', 0) if seed is None else seed + threads = manifest.get('threads') if threads is None else threads + for entry in manifest['entries']: + entry = dict(entry) + name, query_order = entry['id'], entry.pop('query_order', 'relation') + record, _ = validate_training(entry, input_root, manifest['data_root']) + for key in ('reference', 'finalized', 'blockers', 'verification', 'graph_ablation'): + entry.pop(key, None) + entry['root'] = str(input_path(input_root, manifest['data_root'])) + entry['checkpoint'] = str(input_path(input_root, entry['checkpoint'])) + provenance = None + if entry.get('training'): + provenance = dict(training=record, training_sha256=checksum(input_path(input_root, entry.pop('training')))) + entry['options'] = dict(entry.get('options', {})) + if entry['method'] in REFERENCES: + entry['vocabulary_dataset'] = entry['dataset'] + if 'per_shape' in entry['options']: + entry['per_shape'] = entry['options'].pop('per_shape') + else: + entry['adapters'] = {key: str(input_path(input_root, path)) for key, path in entry['adapters'].items()} + data = filter_corrections = None + if manifest.get('answer_filter') == 'corrected' and split == 'test': + from dicee.query_answering.datasets import audit_plus_h_filters, corrected_answer_filters, load_benchmark + data = load_benchmark(entry['root'], entry['dataset'], split='test', query_types=entry.get('query_types'), + inference_graph=entry.get('inference_graph')) + filter_corrections = corrected_answer_filters(data, audit_plus_h_filters(entry['root'], data)) + report = evaluate_method(entry, output=Path(output) / name, device=device, split=split, limit=limit, seed=seed, + threads=threads, query_order=query_order, data=data, filter_corrections=filter_corrections, + provenance=provenance) + print(json.dumps(dict(id=name, queries=report['queries'], seconds=report['seconds'], averages=report['averages'])), flush=True) diff --git a/benchmarks/cqa/inputs.py b/benchmarks/cqa/inputs.py new file mode 100644 index 000000000..b3bc9bb08 --- /dev/null +++ b/benchmarks/cqa/inputs.py @@ -0,0 +1,338 @@ +"""Fetch official benchmark inputs with the standard library; never load models. + +Downloads and extraction use 1 MiB buffers. Existing inputs are checked and +never replaced silently. +""" + +import hashlib +import json +import os +import shutil +import stat +import tempfile +import urllib.error +import urllib.request +import zipfile +from pathlib import Path, PurePosixPath + +from .manifests import REPO, catalog + +CHUNK = 1024 * 1024 +RESERVE = 1024 * 1024 * 1024 +MODEL_URL = ('https://rssiste-my.sharepoint.com/:u:/g/personal/' + 'cosimo_gregucci_ms_informatik_uni-stuttgart_de/' + 'EccKx4K6sZ1Nhy_fJTL8L44BFtpNRZwmI43-ffPRSmb28g?e=kO5dYk&download=1') +MODEL_PREFIX = 'checkpoints/query-baselines/' +# SHA-256 of the released files, extracted from the archive pinned in the +H +# manifests. The standalone weights are from the pinned ULTRA/TRIX trees. +WEIGHT_HASHES = { + 'Experiments/query-baselines/upstream/ultra/ckpts/ultraquery.pth': '9b6dc20801c35e7c9ebe65764acb4ed9cb5bffc6b0a0a727bd48935589e9388d', + 'checkpoints/query-baselines/iscqa-compl-models/models/CLMPT/CLMPT-fb15k237.ckpt': '4ce1e2abeda74199970c9350e3da10d928d34f974f3c1f42ead553a9c13799b9', + 'checkpoints/query-baselines/iscqa-compl-models/models/CLMPT/CLMPT-icews18.ckpt': '1a450d5cd0c726eec10ff302249143894f2fb3329370204b453ec09d447c227d', + 'checkpoints/query-baselines/iscqa-compl-models/models/CLMPT/CLMPT-nell995.ckpt': 'c61fa9089e6233faf38cabe8e2e0ef066eed5d4123871108de3226e21cafcd18', + 'checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/ICEWS18/checkpoint': 'ba873863eeec2abeeb70e9ae484171fbc56a1f5cfd194cb8a7fac24cae03f9f7', + 'checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/fb15k237checkpoint': '631bd1fd3c5d8095eb2c99b78052c3ea4c9c2dd88f7ab044cbb2c610ed547c18', + 'checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/nellcheckpoint': '42b46696588918dba83262dd88b4884d9b6a27c8975f2c6d5a6bc7c65764168f', + 'checkpoints/query-baselines/iscqa-compl-models/models/ConE/FB15k237/checkpoint': 'c576cd8cf55bebc642b6007e079650ccdfd4f627f1c6a4780f5680ba001ef829', + 'checkpoints/query-baselines/iscqa-compl-models/models/ConE/ICEWS18/checkpoint': '44f0f0178e099206ad8ce0dca39815d57df6eed44992698eb6fd9e73ff92f144', + 'checkpoints/query-baselines/iscqa-compl-models/models/ConE/NELL995/checkpoint': 'f0ac18ae0bf3eacff15207d9320e19d010cd4a494e3bae96e8475b891e886787', + 'checkpoints/query-baselines/iscqa-compl-models/models/GNN-QE/model/model_FB15k237.pth': 'cb45202f9aa68da61a59b88721c76b5883274db309d4cf14037b8fc2c72a93b2', + 'checkpoints/query-baselines/iscqa-compl-models/models/GNN-QE/model/model_ICEWS18.pth': '6b8c1fd64cc7ba96fe2e4834634a2721c4d3914658c1d81f61fdd5526d1b5a13', + 'checkpoints/query-baselines/iscqa-compl-models/models/GNN-QE/model/model_NELL995.pth': '086619b977d7b45658c11b4804b6e0e1c92a356cd4f427b4bf5b8ea1ee7016b3', + 'checkpoints/trix/entity_prediction.pth': '8f6e7266093c2d15ad88d41e9825e4b327171143543cf060cfee907d9a890342', + 'checkpoints/ultra_3g.pth': 'fdedc01b0045fc089d2ad5da08569466b7b221691fe978a18e91082bbb133c18', + 'checkpoints/ultra_4g.pth': '48a046e708adf5632d87c30eacae01f5f51466b2301effdc2cb42358d22854e0', + 'checkpoints/ultra_50g.pth': 'f1c5377b2cf547aaa67520ffb6fce27b75b6eb3417b86dc9cf9dba89964ed10f', + 'checkpoints/kgicl/KG-ICL-6L/model_best.tar': '80f885ffa2c0175cd6ed829af99ae45177fdb5f77302c7648591bbdf62bc1c81', +} + + +def weight_urls(): + ultra, trix, kgicl = (f'https://raw.githubusercontent.com/{repository}/{commit}' + for repository, commit in (catalog().ULTRA, catalog().TRIX, catalog().KG_ICL)) + return {'Experiments/query-baselines/upstream/ultra/ckpts/ultraquery.pth': f'{ultra}/ckpts/ultraquery.pth', + 'checkpoints/kgicl/KG-ICL-6L/model_best.tar': f'{kgicl}/checkpoint/KG-ICL-6L/model_best.tar', + 'checkpoints/ultra_3g.pth': f'{ultra}/ckpts/ultra_3g.pth', + 'checkpoints/ultra_4g.pth': f'{ultra}/ckpts/ultra_4g.pth', + 'checkpoints/ultra_50g.pth': f'{ultra}/ckpts/ultra_50g.pth', + 'checkpoints/trix/entity_prediction.pth': f'{trix}/entity_prediction.pth'} + + +def digest(path): + result = hashlib.sha256() + with Path(path).open('rb') as stream: + while block := stream.read(CHUNK): + result.update(block) + return result.hexdigest() + + +def target(root, relative): + name = PurePosixPath(relative) + if name.is_absolute() or '..' in name.parts or '\\' in relative: + raise ValueError(f'Unsafe input path: {relative}') + result = root / relative + if not result.resolve().is_relative_to(root.resolve()) or result.is_symlink(): + raise ValueError(f'Unsafe input path: {relative}') + return result + + +def space(root, required): + available = shutil.disk_usage(root).free + if available < required + RESERVE: + raise OSError(f'Need {required / 2**30:.2f} GiB plus 1 GiB reserve; ' + f'only {available / 2**30:.2f} GiB free at {root}') + + +def install(stream, path, expected=None): + """Write atomically, rejecting a different existing file or bad checksum.""" + path.parent.mkdir(parents=True, exist_ok=True) + handle, temporary = tempfile.mkstemp(prefix='.input-', dir=path.parent) + temporary = Path(temporary) + result = hashlib.sha256() + try: + with os.fdopen(handle, 'wb') as output: + while block := stream.read(CHUNK): + result.update(block) + output.write(block) + checksum = result.hexdigest() + if expected is not None and checksum != expected: + raise ValueError(f'SHA-256 mismatch for {path}') + if path.exists(): + if digest(path) != checksum: + raise ValueError(f'Existing input differs; refusing to replace {path}') + else: + os.replace(temporary, path) + return checksum + finally: + temporary.unlink(missing_ok=True) + + +def download(url, path, expected=None, size=None): + print(f'Downloading {url}', flush=True) + request = urllib.request.Request(url, headers={'User-Agent': 'dicee-benchmark-inputs/1'}) + with urllib.request.urlopen(request, timeout=120) as response: + length = size or int(response.headers.get('Content-Length') or 0) + if length: + space(path.parent, length) + return install(response, path, expected) + + +def transductive_files(): + return {'id2ent.pkl', 'id2rel.pkl', 'train.txt', 'valid.txt', 'test.txt', + *(f'{split}-{kind}.pkl' for split in ('valid', 'test') + for kind in ('queries', 'easy-answers', 'hard-answers'))} + + +def plus_h_files(icews=False): + files = transductive_files() + if icews: + files -= {'train.txt', 'valid.txt', 'test.txt'} + files |= {f'KG_splits/{split}.txt' for split in ('train', 'valid', 'test')} + else: + files |= {f'test-query-reduction/{shape}/all/test-{kind}.pkl' + for shape in ('2in', 'pni') for kind in ('queries', 'easy-answers', 'hard-answers')} + reductions = {'ip': ('1p', '2i', '2p', 'ip'), 'pi': ('1p', '2i', '2p', 'pi'), + 'up': ('1p', '2u', 'up')} + reductions.update({f'{n}{op}': ('1p', *(f'{i}{op}' for i in range(2, n + 1))) + for op in ('p', 'i') for n in (2, 3, 4)}) + reductions.update({s: ('pos-exist', 'pos-only-miss') for s in ('3in', 'pin', 'inp')}) + files |= {f'test-query-reduction/{shape}/{label}/test-hard-answers.pkl' + for shape, labels in reductions.items() for label in labels} + return files + + +def plan(manifest, suite): + """Dataset archives and checkpoints that a resolved manifest needs.""" + c = catalog() + names = {entry['dataset'] for entry in manifest['entries']} + weights = sorted({entry['checkpoint'] for entry in manifest['entries']}) + root = manifest['data_root'] + if suite == 'plus_h': + archive = json.loads((REPO / 'benchmarks/plus_h/baselines.json').read_text())['archives']['datasets'] + folders = {f'{c.PLUS_H_PREFIX}/{folder}': plus_h_files(name == 'ICEWS18+H') + for name, folder in c.PLUS_H_FOLDERS.items() if name in names} + return [dict(id='plus-h-datasets', url=c.PLUS_H_ARCHIVE, root=root, folders=folders, + sha256=archive['sha256'], size=archive['bytes'], selective=True)], weights + jobs = [] + folders = {folder: transductive_files() for name, folder in c.TRANSDUCTIVE.items() if name in names} + if folders: + jobs.append(dict(id='ultra-transductive', url='https://snap.stanford.edu/betae/KG_data.zip', root=root, folders=folders)) + required = {f'{split}_{kind}.pkl' for split in ('valid', 'test') for kind in ('queries', 'answers_easy', 'answers_hard')} + for version in c.INDUCTIVE_VERSIONS: + if f'InductiveFB15k237Query:{version}' in names: + jobs.append(dict(id=f'ultra-inductive-{version}', url=f'https://zenodo.org/records/7306046/files/{version}.zip', root=root, + folders={version: required | {'train_graph.txt', 'val_inference.txt', 'test_inference.txt'}})) + folders = {f'WikiTopics_QE/{topic}': required | {'train_graph.txt', 'test_inference.txt', 'og_mappings.pkl'} + for topic in c.WIKITOPICS if f'WikiTopicsQuery:{topic}' in names} + if folders: + jobs.append(dict(id='ultra-wikitopics', url='https://reltrans.s3.us-east-2.amazonaws.com/WikiTopics_QE.zip', + root=root, folders=folders)) + return jobs, weights + + +def missing_files(root, job, test_labels=True): + """Required files of a dataset job that are absent; ``test_labels=False`` skips +H test reduction labels.""" + return [f"{job['root']}/{folder}/{name}" for folder, names in job['folders'].items() + for name in sorted(names) if (test_labels or not name.startswith('test-query-reduction/')) + and not target(root, f"{job['root']}/{folder}/{name}").is_file()] + + +def extract(archive, root, select): + with zipfile.ZipFile(archive) as zipped: + members = zipped.infolist() + # Validate the entire archive before installing any input. + for member in members: + target(root, member.filename) + if stat.S_ISLNK(member.external_attr >> 16): + raise ValueError(f'Symlink in archive: {member.filename}') + chosen = [(m, select(m.filename)) for m in members if not m.is_dir()] + chosen = [(m, target(root, name)) for m, name in chosen if name is not None] + space(root, sum(m.file_size for m, _ in chosen)) + checksums = {} + for member, path in chosen: + with zipped.open(member) as stream: + checksums[path.relative_to(root).as_posix()] = install(stream, path) + return checksums + + +def fetch_dataset(root, job): + state = root / '.benchmark-downloads' + state.mkdir(parents=True, exist_ok=True) + receipt = state / f"{job['id']}.json" + if not missing_files(root, job): + if not receipt.is_file(): + print(f"Dataset inputs already present: {job['id']} (freeze pins their hashes)", flush=True) + return + saved = json.loads(receipt.read_text()) + if saved['url'] == job['url'] and all(target(root, p).is_file() and digest(target(root, p)) == h + for p, h in saved['files'].items()): + print(f"Verified dataset {job['id']}", flush=True) + return + raise ValueError(f"Dataset receipt mismatch: {receipt}") + with tempfile.TemporaryDirectory(prefix='.fetch-', dir=root) as temporary: + archive = Path(temporary) / 'data.zip' + checksum = download(job['url'], archive, job.get('sha256'), job.get('size')) + + def select(name): + for folder, required in job['folders'].items(): + if name.startswith(folder + '/'): + relative = name[len(folder) + 1:] + if not job.get('selective') or relative in required or relative == 'stats.txt': + return f"{job['root']}/{name}" + return None + + files = extract(archive, root, select) + missing = missing_files(root, job) + if missing: + raise ValueError(f"Archive lacks required inputs: {missing}") + payload = json.dumps(dict(url=job['url'], archive_sha256=checksum, files=files), indent=2).encode() + # A receipt is metadata, not an immutable benchmark input. + temp = receipt.with_suffix('.tmp') + temp.write_bytes(payload) + os.replace(temp, receipt) + print(f"Installed {len(files)} files for {job['id']}", flush=True) + + +def fetch_weights(root, weights, models_archive=None): + urls = weight_urls() + pending = [] + for name in weights: + path = target(root, name) + if path.is_file(): + if name in WEIGHT_HASHES and digest(path) != WEIGHT_HASHES[name]: + raise ValueError(f'Existing checkpoint checksum mismatch: {path}') + else: + if name not in WEIGHT_HASHES: + raise ValueError(f'No public download is configured for {path}; provide this custom checkpoint first') + pending.append(name) + author = [name for name in pending if name not in urls] + if author: + meta = json.loads((REPO / 'benchmarks/plus_h/baselines.json').read_text())['archives']['models'] + with tempfile.TemporaryDirectory(prefix='.models-', dir=root) as temporary: + archive = models_archive or Path(temporary) / 'models.zip' + if models_archive: + if digest(archive) != meta['sha256']: + raise ValueError(f'Official model archive checksum mismatch: {archive}') + else: + try: + download(MODEL_URL, archive, meta['sha256'], meta['bytes']) + except urllib.error.HTTPError as error: + raise RuntimeError('The authors\' model host rejected the download. ' + 'Use --models-archive /path/to/iscqa-compl-models.zip, ' + 'or copy the released checkpoints into the manifest paths and retry.') from error + wanted = {name[len(MODEL_PREFIX):]: name for name in author} + extract(archive, root, wanted.get) + for name in pending: + path = target(root, name) + if name in urls: + download(urls[name], path, WEIGHT_HASHES[name]) + if not path.is_file() or digest(path) != WEIGHT_HASHES[name]: + raise ValueError(f'Released checkpoint missing or checksum mismatch: {path}') + print(f'Checkpoint inputs present: {len(weights)}; released files checked by SHA-256', flush=True) + + +def ensure_inputs(manifest, input_root, *, suite, models_archive=None, download_missing=True, test_labels=True): + """Set up only requested public inputs; custom data/checkpoints stay explicit. + + ``test_labels=False`` (validation-only evaluation) neither requires nor downloads + the +H test reduction labels, which only test-split difficulty reports read. + """ + root = Path(input_root).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + jobs, weights = plan(manifest, suite) + missing = [p for p in weights if not target(root, p).is_file()] + public = json.loads((REPO / 'benchmarks' / suite / 'baselines.json').read_text()) + official = manifest.get('archives', {}).get('datasets') == public['archives']['datasets'] + if official: + missing += [p for job in jobs for p in missing_files(root, job, test_labels)] + extras = {p for e in manifest['entries'] for p in e.get('adapters', {}).values()} + extras |= {e['checkpoint'] for e in manifest['entries'] if e['checkpoint'].endswith('.json')} + for name in sorted(extras): + path = target(root, name) + if not path.is_file(): + source = target(REPO, name) + if not source.is_file() or not download_missing: + missing.append(name) + continue + space(root, source.stat().st_size) + with source.open('rb') as stream: + install(stream, path) + weights = [name for name in weights if name not in extras] + missing = [p for p in missing if not target(root, p).is_file()] + if not download_missing: + if missing: + raise FileNotFoundError(f'Inputs missing with --no-setup: {missing}') + return + if weights: + fetch_weights(root, weights, models_archive) + if official: + for job in jobs: + if missing_files(root, job, test_labels): + fetch_dataset(root, job) + + +def describe(manifest, root, *, suite): + """Print where each input of ``manifest`` comes from; no downloads or writes.""" + jobs, weights = plan(manifest, suite) + urls = weight_urls() + for job in jobs: + print(f"{job['id']}: {job['url']} -> {root / job['root']}") + for name in weights: + source = urls.get(name, MODEL_URL if name in WEIGHT_HASHES else 'User-provided input') + print(f'{source} -> {root / name}') + + +def check(manifest, root, *, suite): + """Print missing inputs and checksum mismatches; return whether there were any.""" + jobs, weights = plan(manifest, suite) + adapters = {path for entry in manifest['entries'] for path in entry.get('adapters', {}).values()} + missing = [path for job in jobs for path in missing_files(root, job)] + missing += [path for path in [*weights, *sorted(adapters)] if not target(root, path).is_file()] + bad = [path for path in weights if path in WEIGHT_HASHES and target(root, path).is_file() + and digest(target(root, path)) != WEIGHT_HASHES[path]] + for name in missing: + print(f'Missing: {name}') + for name in bad: + print(f'Checksum mismatch: {name}') + print(f'{len(jobs)} dataset archives; {len(weights)} checkpoints; {len(missing)} missing files; {len(bad)} bad checkpoints.') + return bool(missing or bad) diff --git a/benchmarks/cqa/manifests.py b/benchmarks/cqa/manifests.py new file mode 100644 index 000000000..b705bccb2 --- /dev/null +++ b/benchmarks/cqa/manifests.py @@ -0,0 +1,273 @@ +"""Benchmark suites and their public recipes, resolved without PyTorch. + +A recipe (manifest entry) fixes a method, checkpoint, dataset, query types and +execution options. Resolution only selects and annotates entries; it never +opens datasets or models, so the host launcher can run it before Docker. + +Public manifests group entries into ``recipes``: each entry is the recipe's +``defaults`` updated by the entry's own fields, recursively for objects, and +``{dataset}`` in an ID is replaced by the entry's dataset. Manifests may also +list resolved ``entries`` directly. +""" + +import copy +import importlib.util +import json +from functools import cache +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +SUITES = { + 'plus_h': dict(title='+H benchmark', answer_filter='corrected'), + 'ultraquery': dict(title='UltraQuery benchmark (released PyG)', answer_filter='released'), +} +KGFM_EXECUTION = { + # ULTRA/TRIX adapters on H100 / H100 NVL, as measured on full validation + # splits: atomic batches of 16 and a per-row cache of raw backbone scores + # in host memory, which beams of different queries share. Native methods + # need no profile. + 'h100': dict(row_batch_size=16, backend_batch_size=16, cache_bytes=512 * 2**20, raw_cache_bytes=24 * 2**30, + raw_cache_device='cpu', raw_cache_granularity='row'), +} + + +@cache +def catalog(): + """The library's benchmark catalog, loaded without importing dicee or PyTorch.""" + spec = importlib.util.spec_from_file_location('_dicee_cqa_catalog', REPO / 'dicee/query_answering/catalog.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def suite_directory(suite): + if suite not in SUITES: + raise ValueError(f'Choose a suite: {", ".join(SUITES)}') + return REPO / 'benchmarks' / suite + + +def default_recipes(suite): + """Public recipe files of a suite: native baselines, then KGFM adapters.""" + directory = suite_directory(suite) + return [path for path in (directory / 'baselines.json', directory / 'kgfm_adapters.json') if path.is_file()] + + +def merge(defaults, override): + """``defaults`` updated by ``override``; objects merge recursively, other values are replaced.""" + result = copy.deepcopy(defaults) + for key, value in override.items(): + nested = isinstance(value, dict) and isinstance(result.get(key), dict) + result[key] = merge(result[key], value) if nested else copy.deepcopy(value) + return result + + +def expand(recipes): + """Resolved entries of recipe groups, in order.""" + entries = [] + for recipe in recipes: + for override in recipe['entries']: + entry = merge(recipe['defaults'], override) + entry['id'] = entry['id'].replace('{dataset}', entry['dataset']) + entries.append(entry) + return entries + + +def read_manifest(path): + """A recipe manifest with resolved entries, or the manifest pinned in a frozen bundle.""" + value = json.loads(Path(path).read_text()) + value = value.get('manifest', value) + if 'recipes' in value: + value = {key: item for key, item in value.items() if key != 'recipes'} | {'entries': expand(value['recipes'])} + return value + + +def select_entries(entries, *, entry_ids=None, methods=None, datasets=None, query_types=None, atomic_negation=False): + """Resolve recipe subsets before opening datasets or model checkpoints.""" + shapes_by_dataset = catalog().query_types_for_dataset + all_shapes = catalog().PLUS_H_SHAPES + identifiers = [entry.get('id', f'{entry["method"]}-{entry["dataset"]}') for entry in entries] + if len(set(identifiers)) != len(identifiers) or any(Path(name).name != name or name in ('', '.', '..') for name in identifiers): + raise ValueError('Manifest entry IDs must be unique, plain directory names') + + def selection(values, available, label, allow_all=True): + if values is None or (allow_all and values == ['all']): + return None + if not values or len(set(values)) != len(values): + raise ValueError(f'Select distinct {label}') + missing = set(values) - set(available) + if missing: + raise ValueError(f'Select distinct {label}; unknown: {", ".join(sorted(missing))}. Available: {", ".join(sorted(available))}') + return set(values) + + ids = selection(entry_ids, identifiers, 'entry IDs', allow_all=False) + wanted_methods = selection(methods, {e['method'] for e in entries}, 'methods') + wanted_datasets = selection(datasets, {e['dataset'] for e in entries}, 'datasets') + selected = [copy.deepcopy(entry) for name, entry in zip(identifiers, entries) + if (ids is None or name in ids) and (wanted_methods is None or entry['method'] in wanted_methods) + and (wanted_datasets is None or entry['dataset'] in wanted_datasets)] + if not selected: + raise ValueError('No manifest entries match the selected methods, datasets, and entry IDs') + groups = {'all', 'epfo', 'negation'} + if query_types is not None: + if not query_types or len(set(query_types)) != len(query_types): + raise ValueError('Select distinct query types or groups: all, epfo, negation') + unknown = set(query_types) - set(all_shapes) - groups + if unknown or ('all' in query_types and len(query_types) != 1): + raise ValueError('Select supported query types or groups: all, epfo, negation; use all alone. ' + f'Available types: {", ".join(all_shapes)}') + for entry in selected: + changed = False + if query_types is not None: + available = shapes_by_dataset(entry['dataset']) + expanded_groups = {'all': available, 'epfo': [s for s in available if 'n' not in s], + 'negation': [s for s in available if 'n' in s]} + shapes = [] + for shape in query_types: + shapes.extend(expanded_groups.get(shape, [shape])) + shapes = list(dict.fromkeys(shapes)) + if set(shapes) - set(available): + raise ValueError(f'Query types {shapes} are not supported by {entry["dataset"]}') + changed = shapes != entry.get('query_types') + entry['query_types'] = shapes + for owner in (entry, entry.get('options', {})): + if 'per_shape' in owner: + owner['per_shape'] = {s: value for s, value in owner['per_shape'].items() if s in shapes} + if 'operators' in entry: + missing = set(shapes) - entry['operators'].keys() + if missing: + raise ValueError(f'{entry.get("id", entry["method"])} has no operator/adapter recipe for {sorted(missing)}') + entry['operators'] = {s: entry['operators'][s] for s in shapes} + if entry['method'] in ('cqd', 'cqd-hybrid'): + options = entry.setdefault('options', {}) + if atomic_negation and not options.get('atomic_negation'): + options['atomic_negation'] = True + changed = True + if any('n' in s for s in entry.get('query_types', [])) and not options.get('atomic_negation'): + raise ValueError('CQD negated query types require atomic_negation=true or --atomic-negation') + if changed and options.get('atomic_negation') and 'reference' in entry: + entry['reference'].setdefault('notes', []).append( + 'CQD-A signed-atom negation with +H scoring.') + if changed: + entry.pop('verification', None) + if 'reference' in entry: + entry['reference']['status'] = 'custom query selection or executor; independent validation required' + return selected + + +def validate_entry(entry): + required = {'id', 'method', 'dataset', 'checkpoint', 'options', 'query_types', 'query_batch_size', 'query_order', 'reference'} + optional = {'adapters', 'operators', 'selection_protocol', 'finalized', 'blockers', 'verification', 'adapter_ablation', + 'inference_graph', 'graph_ablation', 'training'} + if required - entry.keys() or entry.keys() - required - optional: + raise ValueError(f'Invalid benchmark entry fields: {entry.get("id")}') + if (entry['method'] not in catalog().METHODS + or Path(entry['id']).name != entry['id'] or entry['id'] in ('', '.', '..')): + raise ValueError('Invalid benchmark method, dataset, or entry ID') + shapes = entry['query_types'] + if not shapes or len(set(shapes)) != len(shapes) or set(shapes) - set(catalog().query_types_for_dataset(entry['dataset'])): + raise ValueError('Invalid query types for the selected dataset') + catalog().check_recipe(entry, shapes) + if entry['query_order'] not in ('canonical', 'relation', 'upstream-pickle'): + raise ValueError('Choose canonical, relation, or upstream-pickle query ordering') + if entry.get('inference_graph') not in (None, 'train', 'train+valid'): + raise ValueError('Inference graph must be train or train+valid') + if entry.get('graph_ablation') is not None and not isinstance(entry['graph_ablation'], str): + raise ValueError('Graph ablation must identify the shared frozen recipe') + if entry['method'] == 'clmpt' and entry['query_order'] != 'upstream-pickle': + raise ValueError('CLMPT requires frozen reference conversion order') + + +def prepare_manifest(paths, *, profile='bounded', entries=None, methods=None, datasets=None, query_types=None, + atomic_negation=False, observed_facts=None, inference_graphs=None, answer_filter='corrected', suite='plus_h', + hardware_profile='default', kgfm_batch_size=None): + """Merge public recipes without changing beams or per-shape operator settings.""" + manifests = [read_manifest(path) for path in paths] + if not manifests or profile not in ('bounded', 'reference'): + raise ValueError('Choose a bounded or reference profile and at least one manifest') + if hardware_profile not in ('default', *KGFM_EXECUTION): + raise ValueError('Choose a default or h100 hardware profile') + if kgfm_batch_size is not None and (type(kgfm_batch_size) is not int or kgfm_batch_size < 1): + raise ValueError('KGFM batch size must be a positive integer') + if answer_filter not in ('released', 'corrected'): + raise ValueError('Answer filter must be released or corrected') + if suite not in SUITES: + raise ValueError(f'Choose a suite: {", ".join(SUITES)}') + if suite == 'ultraquery' and (answer_filter != 'released' or inference_graphs is not None): + raise ValueError('UltraQuery uses released answer filters and dataset-defined inference graphs') + manifest = {key: value for key, value in manifests[0].items() if key != 'entries'} + combined = [] + for source in manifests: + if {key: value for key, value in source.items() if key != 'entries'} != manifest: + raise ValueError('Source manifests disagree on dataset, seed, threads, or archive metadata') + combined.extend(source['entries']) + if observed_facts is not None: + modes = ('none', 'atomic', 'both') if observed_facts == ['all'] else observed_facts + if not modes or len(modes) != len(set(modes)) or set(modes) - {'none', 'atomic', 'both'}: + raise ValueError('Select distinct observed-fact modes, or all') + variants = [] + for entry in combined: + if not entry['method'].endswith('-adapter'): + continue + for mode in modes: + variant = copy.deepcopy(entry) + variant['id'] += f'-facts-{mode}' + variant['options']['observed_facts'] = mode + variant['reference']['status'] = 'observed-fact ablation; verification pending' + variant['reference'].setdefault('notes', []).append( + f'Inference observed-fact mode: {mode}. Keep the trained weights and graph inputs fixed; ' + 'apply this mode to both learned and identity calibration. No adapter refitting.') + variants.append(variant) + if not variants: + raise ValueError('Observed-fact ablation requires KGFM entries') + combined = variants + if inference_graphs is not None: + if (not inference_graphs or len(set(inference_graphs)) != len(inference_graphs) + or set(inference_graphs) - {'train', 'train+valid'}): + raise ValueError('Select distinct inference graphs: train, train+valid') + variants = [] + for entry in combined: + if entry['method'] in catalog().GRAPH_INDEPENDENT_METHODS: + entry['inference_graph'] = 'train' + entry['reference']['graph_independent'] = True + variants.append(entry) + continue + for graph in inference_graphs: + variant = copy.deepcopy(entry) + variant['graph_ablation'] = entry['id'] + variant['inference_graph'] = graph + variant['id'] += '-graph-' + graph.replace('+', '-') + variants.append(variant) + combined = variants + ids = [entry['id'] for entry in combined] + if len(ids) != len(set(ids)): + raise ValueError('Source manifests contain duplicate entry IDs') + manifest['entries'] = select_entries(combined, entry_ids=entries, methods=methods, datasets=datasets, + query_types=query_types, atomic_negation=atomic_negation) + allowed_datasets = catalog().PLUS_H_DATASETS if suite == 'plus_h' else catalog().BENCHMARK_DATASETS + if any(entry['dataset'] not in allowed_datasets for entry in manifest['entries']): + raise ValueError(f'Selected entries must belong to the {suite} dataset suite') + manifest['answer_filter'] = answer_filter + kgfm_methods = catalog().KGFM_ADAPTERS + if kgfm_batch_size is not None and not any(e['method'] in kgfm_methods for e in manifest['entries']): + raise ValueError('KGFM batch size requires at least one selected KGFM adapter') + for entry in manifest['entries']: + entry.pop('verification', None) + if profile == 'bounded' and entry['method'] in ('cqd', 'cqd-hybrid'): + entry['options'].update(reference_batching=False, row_batch_size=32, final_batch_size=32) + entry['reference']['status'] = 'bounded execution profile; independent validation required' + entry['reference'].setdefault('notes', []).append( + 'Bounded profile: reference_batching=false, row_batch_size=32, and final_batch_size=32; beams and per-shape ' + 'recipes retained. Requires independent bounded-profile validation scores, orders, and ties.') + kgfm = entry['method'] in kgfm_methods + if hardware_profile != 'default' and kgfm: + entry['options'].update(KGFM_EXECUTION[hardware_profile]) + entry['reference']['status'] = 'H100 execution profile; independent validation required' + entry['reference'].setdefault('notes', []).append( + 'H100 / H100 NVL configuration measured on full validation splits. Float32 IEEE, query batches, beams, ' + 'operators and observed facts retained. Validate memory, scores, orders and ties on the target GPU.') + if kgfm and kgfm_batch_size is not None: + entry['options'].update(row_batch_size=kgfm_batch_size, backend_batch_size=kgfm_batch_size) + entry['reference']['status'] = 'custom KGFM atomic batch; independent validation required' + entry['reference'].setdefault('notes', []).append( + f'Atomic row and backbone batch size {kgfm_batch_size}; frozen before verification.') + return manifest diff --git a/benchmarks/cqa/oracles.py b/benchmarks/cqa/oracles.py new file mode 100644 index 000000000..c053493f0 --- /dev/null +++ b/benchmarks/cqa/oracles.py @@ -0,0 +1,135 @@ +"""Validate independently exported reference scores on the intended hardware.""" + +import tempfile +from collections import Counter, defaultdict + +import torch + +from dicee.query_answering._checkpoint import write_json +from dicee.query_answering.benchmark import METRICS, QueryMetrics +from dicee.query_answering.context import fingerprint +from dicee.query_answering.datasets import BENCHMARK_DATASETS +from dicee.query_answering.methods import REFERENCES + +from .study import bundle_entry, checksum, dataset_key, entry_inputs_identity, run_job, verify_bundle + +# Query types an upstream oracle cannot cover; the exporter declares exactly these, with this reason. +NEGATED_TYPES = ('2in', '3in', 'inp', 'pin', 'pni') +EXEMPTION_REASON = ('No upstream reference: the pinned +H CQD has no negation in its beam search. The negated types use CQD-A ' + 'signed-atom negation, checked against CQD-A outputs in ' + 'tests/test_query_baselines.py::test_opt_in_cqd_negation_matches_cqda.') + + +def upstream_exempt_types(entry): + """The pinned +H CQD has no negation in its beam search, so CQD-A negated types have no upstream reference.""" + if entry['method'] in ('cqd', 'cqd-hybrid') and entry['options'].get('atomic_negation'): + return sorted(set(entry['query_types']) & set(NEGATED_TYPES)) + return [] + + +def verify_predictions(bundle_dir, entry_id, input_root, reference, output, *, device='cpu', atol=3e-5, rtol=3e-5, + comparison_reference=None): + bundle = verify_bundle(bundle_dir, input_root) + entry = bundle_entry(bundle, entry_id) + oracle = torch.load(reference, map_location='cpu', weights_only=True) + expected = dict(version=1, entry_sha256=fingerprint({key: value for key, value in entry.items() if key != 'verification'}), + inputs_sha256=entry_inputs_identity(bundle, entry), + validation_plan_sha256=bundle['plans'][f'{entry_id}/valid']['sha256']) + if any(oracle.get(key) != value for key, value in expected.items()): + raise ValueError('Reference does not match the frozen entry, inputs, and validation batches') + if entry['method'] in REFERENCES and oracle.get('reference_commit') != REFERENCES[entry['method']][1]: + raise ValueError('Reference commit mismatch') + graphs = {split: bundle['datasets'][dataset_key(entry, split)]['context'] for split in ('valid', 'test')} + if (entry['method'] in REFERENCES and ('inference_graph' in entry or entry['dataset'] in BENCHMARK_DATASETS) + and oracle.get('graphs') != graphs): + raise ValueError('Reference inference graphs differ from the frozen validation/test protocol') + exempt = set(upstream_exempt_types(entry)) + if set((oracle.get('exempt_types') or {}).get('types', [])) != exempt: + raise ValueError('Reference exemptions differ from the query types its upstream code cannot answer') + skipped = Counter() + control = None + comparisons = {} + if comparison_reference is not None: + control = torch.load(comparison_reference, map_location='cpu', weights_only=True) + fields = ('inputs_sha256', 'validation_plan_sha256', 'reference_commit', 'environment') + if any(control.get(key) != oracle.get(key) for key in fields): + raise ValueError('Comparison reference uses different inputs, query batches, upstream code, or hardware') + if set(control['scores']) != set(oracle['scores']) or set(control['orders']) != set(oracle['orders']): + raise ValueError('Comparison reference query coverage differs') + results = {} + + def compare(query, actual): + if query.shape in exempt: + skipped[query.shape] += 1 # evaluated, but without an upstream reference + return + key = query.identity + gold = oracle['scores'][key].to(actual.device) + order = oracle['orders'][key].to(actual.device) + if gold.shape != actual.shape or order.shape != actual.shape: + raise ValueError('Reference must score and rank every candidate') + if not torch.equal(order.sort().values, torch.arange(len(order), device=order.device)): + raise ValueError('Reference order is not a candidate permutation') + if not bool((gold[order][:-1] >= gold[order][1:]).all()): + raise ValueError('Reference order contradicts its scores') + scores_match = torch.allclose(actual, gold, atol=atol, rtol=rtol) + ranks_match = torch.equal(actual.argsort(descending=True), order) + ties_match = torch.equal(actual[order][1:] == actual[order][:-1], gold[order][1:] == gold[order][:-1]) + finite = torch.isfinite(actual) & torch.isfinite(gold) + errors = (actual[finite] - gold[finite]).abs() + results[key] = dict(shape=query.shape, max_error=float(errors.max()) if errors.numel() else 0., + scores_match=scores_match, ranks_match=ranks_match, ties_match=ties_match, + passed=scores_match and ranks_match and ties_match) + if control is not None: + previous = control['scores'][key].to(actual.device) + previous_order = control['orders'][key].to(actual.device) + if previous.shape != actual.shape or not torch.equal(previous.argsort(descending=True), previous_order): + raise ValueError('Comparison reference has invalid candidate scores or order') + finite = torch.isfinite(actual) & torch.isfinite(previous) + errors = (actual[finite] - previous[finite]).abs() + metric = QueryMetrics(len(actual)) + current_metrics = metric.evaluate(actual, query.easy, query.hard)[0] + previous_metrics = metric.evaluate(previous, query.easy, query.hard)[0] + comparisons[key] = dict(shape=query.shape, max_error=float(errors.max()) if errors.numel() else 0., + ranks_match=torch.equal(actual.argsort(descending=True), previous_order), + ties_match=torch.equal(actual[previous_order][1:] == actual[previous_order][:-1], + previous[previous_order][1:] == previous[previous_order][:-1]), + current=current_metrics, comparison=previous_metrics) + + with tempfile.TemporaryDirectory(prefix='dicee-cqa-parity-') as temporary: + report = run_job(bundle_dir, entry_id, input_root, temporary, device=device, phase='pilot', + pilot_queries=oracle['pilot_queries'], on_prediction=compare, + probe_only=oracle.get('probe_only', False)) + env = report['benchmark_run']['environment'] + hardware_keys = ('python', 'torch', 'cuda', 'gpu', 'driver', 'precision', 'threads') + if any(oracle.get('environment', {}).get(key) != env[key] for key in hardware_keys): + raise ValueError('Reference was produced with different hardware, precision, or runtime versions') + if set(results) != set(oracle['scores']) or set(results) != set(oracle['orders']): + raise ValueError('Reference query coverage differs from the frozen pilot batches') + if {row['shape'] for row in results.values()} != set(entry['query_types']) - exempt: + raise ValueError('Reference omits a requested query type') + if set(skipped) != exempt: + raise ValueError('An exempt query type was not evaluated') + evidence = dict(entry_sha256=expected['entry_sha256'], inputs_sha256=expected['inputs_sha256'], + source_sha256=bundle['source_sha256'], environment=env, reference_sha256=checksum(reference), + reference_commit=oracle.get('reference_commit'), queries=results, + graphs=graphs, + reference_adjustments=oracle.get('runtime_adjustments', []), + probe_only=oracle.get('probe_only', False), + scope='paired full-candidate validation predictions, exact frozen batch membership', + exempt_types=dict(types=sorted(exempt), reason=EXEMPTION_REASON, evaluated_queries=dict(skipped)) if exempt else None, + atol=atol, rtol=rtol, passed=all(value['passed'] for value in results.values())) + if control is not None: + groups = defaultdict(list) + for row in comparisons.values(): + groups[row['shape']].append(row) + differences = {shape: {policy: {metric: sum(row['current'][policy][metric] - row['comparison'][policy][metric] + for row in rows) / len(rows) for metric in METRICS} for policy in ('sort', 'expected')} + for shape, rows in groups.items()} + evidence['upstream_comparison'] = dict(reference_sha256=checksum(comparison_reference), + comparison_entry_sha256=control.get('entry_sha256'), queries=comparisons, + delta_per_type=differences, + delta_macro={policy: {metric: sum(row[policy][metric] for row in differences.values()) / len(differences) + for metric in METRICS} for policy in ('sort', 'expected')}, + scope='Diagnostic comparison only; declared reference scores, ranks and ties remain the verification gate.') + write_json(output, evidence) + return evidence diff --git a/benchmarks/cqa/provenance.py b/benchmarks/cqa/provenance.py new file mode 100644 index 000000000..e13e224a1 --- /dev/null +++ b/benchmarks/cqa/provenance.py @@ -0,0 +1,60 @@ +"""Bind locally trained checkpoints to their recorded dataset and run profile.""" + +from pathlib import Path + +from dicee.query_answering._checkpoint import checksum, input_path, read +from dicee.query_answering.datasets import dataset_spec + +REQUIRED_INPUTS = { + 'qto': {'id2ent.pkl', 'id2rel.pkl', 'train.txt', 'valid.txt', 'test.txt'}, + 'inductive-gnnqe': {'train_graph.txt', 'val_inference.txt', 'test_inference.txt', + 'train_queries.pkl', 'train_answers_hard.pkl'} + | {f'{split}_{kind}.pkl' for split in ('valid', 'test') + for kind in ('queries', 'answers_easy', 'answers_hard')}, +} + + +def validate_training(entry, input_root, data_root, *, final=False, file_hashes=None): + """Return the validated record and dataset pins; allow smoke only outside final runs.""" + path = entry.get('training') + if not path: + if (entry['method'] == 'inductive-gnnqe' + or (entry['method'] == 'qto' and entry['dataset'] == 'FB15kLogicalQuery')): + raise ValueError('Training provenance is required for this locally trained baseline') + return None, {} + + def digest(relative): + if file_hashes is not None and str(relative) in file_hashes: + return file_hashes[str(relative)] + return checksum(input_path(input_root, relative)) + + record = read(input_path(input_root, path)) + if (not isinstance(record, dict) or record.get('version') != 1 or record.get('status') != 'complete' + or record.get('method') != entry['method'] or record.get('dataset') != entry['dataset'] + or record.get('checkpoint_sha256') != digest(entry['checkpoint'])): + raise ValueError('Training provenance does not match the completed checkpoint') + if record.get('profile') not in ('author-recipe', 'custom', 'smoke', 'fixture'): + raise ValueError('Training provenance requires a recognized training profile') + releases = {'qto': ('BetaE', 'fixture'), 'inductive-gnnqe': ('InductiveQE v2.0', 'fixture')} + if record.get('dataset_release') not in releases.get(entry['method'], ()): + raise ValueError('Training provenance requires the supported dataset release') + if final and (record['profile'] in ('smoke', 'fixture') or record.get('dataset_release') == 'fixture'): + raise ValueError('Entry is not ready for final testing: smoke/fixture training checkpoint') + inputs = record.get('inputs') + required = REQUIRED_INPUTS.get(entry['method']) + if not required or not isinstance(inputs, dict) or required - inputs.keys(): + raise ValueError('Training provenance is missing required dataset input bindings') + prefix = Path(data_root) / dataset_spec(entry['dataset'])[1] + pins = {} + for name, expected in inputs.items(): + if not isinstance(name, str) or Path(name).name != name or name in ('', '.', '..'): + raise ValueError('Training inputs must name files within the dataset folder') + relative = str(prefix / name) + try: + actual = digest(relative) + except FileNotFoundError as error: + raise ValueError(f'Training input is missing: {relative}') from error + if actual != expected: + raise ValueError(f'Training input differs from the evaluation dataset: {relative}') + pins[relative] = actual + return record, pins diff --git a/benchmarks/cqa/reports.py b/benchmarks/cqa/reports.py new file mode 100644 index 000000000..6df5b0cf1 --- /dev/null +++ b/benchmarks/cqa/reports.py @@ -0,0 +1,487 @@ +"""Rebuild paired tie-policy tables from compact answer ranks.""" + +import csv +import json +import math +import multiprocessing +import re +import sqlite3 +from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor +from contextlib import contextmanager +from functools import partial +from pathlib import Path + +import numpy as np + +from dicee.query_answering._checkpoint import checksum, read, write_json +from dicee.query_answering._query import PLUS_H_SHAPES +from dicee.query_answering.benchmark import METRICS, _averages +from dicee.query_answering.datasets import query_types_for_dataset +from dicee.query_answering.method_evaluation import GRAPH_INDEPENDENT_METHODS + + +def trace_queries(path): + with sqlite3.connect(f'{Path(path).resolve().as_uri()}?mode=ro', uri=True) as connection: + harmonic = np.zeros(1, dtype=np.float64) + for query_id, shape, payload in connection.execute('SELECT query_id, shape, answers FROM queries ORDER BY position'): + records = np.asarray(json.loads(payload), dtype=np.int64) + if records.ndim != 2 or records.shape[1] != 4 or not len(records): + raise ValueError('Malformed answer-rank records') + ranks, greater, sizes = records[:, 1:].T + if (greater < 0).any() or (sizes < 1).any() or (ranks <= greater).any() or (ranks > greater + sizes).any(): + raise ValueError('Inconsistent filtered rank / tie block') + last = greater + sizes + if last.max() >= len(harmonic): + harmonic = np.concatenate(([0.], np.cumsum(1. / np.arange(1, last.max() + 1)))) + sort_metrics = dict(zip(METRICS, [np.mean(1. / ranks), *(np.mean(ranks <= k) for k in (1, 3, 10))])) + expected = dict(zip(METRICS, [np.mean((harmonic[last] - harmonic[greater]) / sizes), + *(np.mean(np.minimum(np.maximum(k - greater, 0), sizes) / sizes) for k in (1, 3, 10))])) + yield dict(query_id=query_id, shape=shape, sort=sort_metrics, expected=expected, + hard_answers=len(records), tied_answers=int((sizes > 1).sum()), max_tie_size=int(sizes.max())) + + +def summarize_trace(path, *, bootstrap_samples=2000, seed=0, expected_types=None): + groups = defaultdict(list) + for query in trace_queries(path): + groups[query['shape']].append(query) + if not groups: + raise ValueError('Empty rank trace') + rng = np.random.default_rng(seed) + per_shape, delta_draws = {}, np.zeros(bootstrap_samples) + for shape, queries in sorted(groups.items()): + policies = {policy: {metric: float(np.mean([query[policy][metric] for query in queries])) for metric in METRICS} + for policy in ('sort', 'expected')} + deltas = np.array([query['expected']['mrr'] - query['sort']['mrr'] for query in queries]) + draws = (np.full(bootstrap_samples, deltas[0]) if np.ptp(deltas) == 0 else + np.array([rng.choice(deltas, size=len(deltas), replace=True).mean() for _ in range(bootstrap_samples)])) + delta_draws += draws / len(groups) + hard = sum(query['hard_answers'] for query in queries) + per_shape[shape] = dict(**policies, queries=len(queries), hard_answers=hard, + tied_answer_fraction=sum(query['tied_answers'] for query in queries) / hard, + max_tie_size=max(query['max_tie_size'] for query in queries), + delta_mrr=float(deltas.mean()), + delta_mrr_ci95=np.quantile(draws, [.025, .975]).tolist() if bootstrap_samples else None) + averages = {policy: _averages({shape: values[policy] for shape, values in per_shape.items()}) for policy in ('sort', 'expected')} + return dict(per_shape=per_shape, averages=averages, all_16_types=set(groups) == set(PLUS_H_SHAPES), + all_benchmark_types=set(groups) == set(expected_types) if expected_types is not None else None, + delta_macro_mrr_ci95=np.quantile(delta_draws, [.025, .975]).tolist() if bootstrap_samples else None, + uncertainty=dict(unit='paired query', stratification='query type', resamples=bootstrap_samples, seed=seed)) + + +def adapter_effects(learned_trace, control_trace, *, bootstrap_samples=2000, seed=0): + """Paired learned-minus-identity differences, conditional on frozen weights.""" + learned = {query['query_id']: query for query in trace_queries(learned_trace)} + groups = defaultdict(list) + for control in trace_queries(control_trace): + current = learned.pop(control['query_id'], None) + if current is None or current['shape'] != control['shape'] or current['hard_answers'] != control['hard_answers']: + raise ValueError('Adapter comparison queries or answer counts differ') + groups[control['shape']].append({policy: {metric: current[policy][metric] - control[policy][metric] + for metric in METRICS} for policy in ('sort', 'expected')}) + if learned or not groups: + raise ValueError('Adapter comparison requires identical nonempty query coverage') + rng = np.random.default_rng(seed) + per_shape, draws = {}, {policy: np.zeros(bootstrap_samples) for policy in ('sort', 'expected')} + for shape, queries in sorted(groups.items()): + row = dict(queries=len(queries)) + for policy in draws: + values = np.array([query[policy]['mrr'] for query in queries]) + samples = (np.full(bootstrap_samples, values[0]) if np.ptp(values) == 0 else + np.array([rng.choice(values, size=len(values), replace=True).mean() for _ in range(bootstrap_samples)])) + draws[policy] += samples / len(groups) + row[policy] = {metric: float(np.mean([query[policy][metric] for query in queries])) for metric in METRICS} + row[policy]['mrr_ci95'] = np.quantile(samples, [.025, .975]).tolist() if bootstrap_samples else None + per_shape[shape] = row + macro = {policy: {metric: float(np.mean([row[policy][metric] for row in per_shape.values()])) for metric in METRICS} + | {'mrr_ci95': np.quantile(samples, [.025, .975]).tolist() if bootstrap_samples else None} + for policy, samples in draws.items()} + return dict(per_shape=per_shape, macro=macro, uncertainty=dict(unit='paired query', stratification='query type', + resamples=bootstrap_samples, seed=seed, scope='Conditional on frozen models and adapters; not training-seed uncertainty.')) + + +@contextmanager +def parallel(workers=1): + """``starmap(function, argument_tuples)``: results in order, computed in up to ``workers`` processes. + + Every bootstrap seeds its own generator, so results do not depend on the number of workers. + """ + if type(workers) is not int or workers < 1: + raise ValueError('Use a positive number of workers') + if workers == 1: + yield lambda function, calls: [function(*arguments) for arguments in calls] + return + with ProcessPoolExecutor(workers, mp_context=multiprocessing.get_context('spawn')) as pool: + yield lambda function, calls: [future.result() for future in [pool.submit(function, *arguments) for arguments in calls]] + + +def offline_summary(path, bootstrap_samples=2000, seed=0): + """(result, expected types, offline summary) of one saved run after checking its rank trace; None for other files.""" + report = read(path) + if report.get('benchmark_run', report.get('paper_run')) is None: + return None + trace = Path(path).parent / 'ranks.sqlite3' + if checksum(trace) != report['rank_trace_sha256']: + raise ValueError(f'Rank trace checksum mismatch: {trace}') + expected_types = report.get('dataset_metadata', {}).get('expected_query_types') + if expected_types is None: + expected_types = query_types_for_dataset(report['dataset']) + summary = summarize_trace(trace, bootstrap_samples=bootstrap_samples, seed=seed, expected_types=expected_types) + for shape, values in summary['per_shape'].items(): + for policy in ('sort', 'expected'): + online = report['per_shape'][shape] if policy == 'sort' else report['additional_tie_metrics'][policy]['per_shape'][shape] + if any(not math.isclose(online[metric], values[policy][metric], abs_tol=1e-11) for metric in METRICS): + raise ValueError(f'Offline rank metrics differ from evaluation: {path}') + return report, expected_types, summary + + +def export_reports(results, published, output, *, bootstrap_samples=2000, seed=0, title='CQA benchmark', workers=1): + """Compare matching coverage only; partial means never become full-suite means. + + ``workers`` processes summarize traces and compute effects; the outputs do not depend on it. + """ + with parallel(workers) as starmap: + return _export_reports(results, published, output, bootstrap_samples=bootstrap_samples, seed=seed, title=title, + starmap=starmap) + + +def _trace(detail): + return Path(detail['result']).parent / 'ranks.sqlite3' + + +def _export_reports(results, published, output, *, bootstrap_samples, seed, title, starmap): + published = read(published) if published is not None else {'source': None, 'datasets': {}} + rows, details = [], [] + paths = sorted(Path(results).rglob('result.json')) + effect = partial(adapter_effects, bootstrap_samples=bootstrap_samples, seed=seed) + for path, saved in zip(paths, starmap(offline_summary, [(path, bootstrap_samples, seed) for path in paths])): + if saved is None: + continue + report, expected_types, summary = saved + run = report.get('benchmark_run', report.get('paper_run')) + method = report['inference']['method'] + scores = published['datasets'].get(report['dataset'], {}).get(method, {}) + coverage = summary['all_benchmark_types'] and report['protocol']['full_split'] and report['split'] == 'test' + sort_mrr = summary['averages']['sort']['all']['mrr'] * 100 + expected = summary['averages']['expected']['all']['mrr'] * 100 + paper = sum(scores.values()) / len(expected_types) if set(scores) == set(expected_types) and coverage else None + graph = report.get('dataset_metadata', {}).get('inference_graph', 'dataset-defined') + answer_filter = report['protocol'].get('answer_filter', 'released') + independent = method in GRAPH_INDEPENDENT_METHODS + native_graph = report['reference'].get('inference_graph') + rows.append(dict(id=run['entry'], dataset=report['dataset'], method=method, + inference_graph=graph, answer_filter=answer_filter, + applicable_graphs=['train', 'train+valid'] if independent else [graph], + reference_graph=native_graph, + reference_graph_matches=True if independent else graph == native_graph if native_graph else None, + calibration=report['inference'].get('calibration'), + observed_facts=report['inference'].get('observed_facts'), + execution_profile=report.get('execution_profile', 'unspecified'), + phase=run['phase'], complete_test=coverage, expected_types=list(expected_types), + complete_16_type_test=coverage and summary['all_16_types'], types=len(summary['per_shape']), + published_mrr=paper, sort_mrr=sort_mrr, expected_random_mrr=expected, + tie_delta=expected-sort_mrr, published_delta=sort_mrr-paper if paper is not None else None, + reference=report['reference'])) + details.append(dict(id=run['entry'], dataset=report['dataset'], **summary, + inference_graph=graph, graph_ablation=report.get('graph_ablation') if run['phase'] == 'test' else None, + graph_recipe_sha256=report.get('graph_recipe_sha256'), + # Candidate and context identity, so comparisons across runs stay verifiable from this file. + num_candidates=report.get('num_candidates'), candidate_sha256=report.get('candidate_sha256'), + context_sha256=report.get('context_sha256'), + calibration=report['inference'].get('calibration'), answer_filter=answer_filter, + filter_paired_with=report.get('filter_paired_with'), + paired_with=report.get('paired_with'), published_per_shape=scores if coverage else {}, result=str(path))) + output = Path(output) + write_json(output / 'comparison.json', dict(rows=rows, details=details, published_source=published['source'])) + with (output / 'per-query-type.csv').open('w', newline='') as stream: + writer = csv.writer(stream) + writer.writerow(['entry', 'dataset', 'inference_graph', 'answer_filter', 'type', 'queries', 'published_mrr', 'sort_mrr', 'expected_random_mrr', 'tie_delta', 'tied_answer_fraction']) + for detail in details: + for shape, values in detail['per_shape'].items(): + writer.writerow([detail['id'], detail['dataset'], detail['inference_graph'], detail['answer_filter'], shape, values['queries'], detail['published_per_shape'].get(shape, ''), + values['sort']['mrr'] * 100, values['expected']['mrr'] * 100, values['delta_mrr'] * 100, values['tied_answer_fraction']]) + lines = [f'# {title}', '', 'MRR ×100. Published comparisons require complete test coverage of the dataset’s query types.', '', + 'Graph-independent methods apply to both graph conditions. Published values may use different graph or answer protocols.', '', + '| Entry | Dataset | Inference facts | Filters | Execution | Scope | Types | Published | Sort | Expected random | Tie Δ |', + '|---|---|---|---|---|---|---:|---:|---:|---:|---:|'] + for row in rows: + paper = f'{row["published_mrr"]:.2f}' if row['published_mrr'] is not None else '—' + scope = 'full test' if row['complete_test'] else row['phase'] + ' / partial' + graph = 'independent' if len(row['applicable_graphs']) > 1 else row['inference_graph'] + lines.append(f'| {row["id"]} | {row["dataset"]} | {graph} | {row["answer_filter"]} | {row["execution_profile"]} | {scope} | {row["types"]} | {paper} | ' + f'{row["sort_mrr"]:.2f} | {row["expected_random_mrr"]:.2f} | {row["tie_delta"]:+.2f} |') + (output / 'comparison.md').write_text('\n'.join(lines) + '\n') + lookup = {row['id']: row for row in details} + controls = [control for control in details if control['paired_with'] in lookup] + if any(lookup[control['paired_with']]['answer_filter'] != control['answer_filter'] for control in controls): + raise ValueError('Adapter comparison changed answer filters') + effects = starmap(effect, [(_trace(lookup[control['paired_with']]), _trace(control)) for control in controls]) + pairs = [dict(learned=control['paired_with'], control=control['id'], dataset=control['dataset'], + answer_filter=control['answer_filter'], **values) for control, values in zip(controls, effects)] + write_json(output / 'adapter-effects.json', pairs) + lines = ['# Learned adapter effect', '', + 'Learned minus identity calibration, MRR ×100. Observed facts, query plans, operators and beams are held fixed.', '', + '| Entry | Dataset | Sort Δ | Expected random Δ | Expected random 95% CI |', + '|---|---|---:|---:|---|'] + with (output / 'adapter-effects.csv').open('w', newline='') as stream: + writer = csv.writer(stream) + writer.writerow(['entry', 'dataset', 'type', 'queries', 'sort_delta_mrr', 'expected_delta_mrr']) + for pair in pairs: + ci = pair['macro']['expected']['mrr_ci95'] + interval = f'[{ci[0] * 100:+.2f}, {ci[1] * 100:+.2f}]' if ci is not None else '—' + lines.append(f'| {pair["learned"]} | {pair["dataset"]} | {pair["macro"]["sort"]["mrr"] * 100:+.2f} | ' + f'{pair["macro"]["expected"]["mrr"] * 100:+.2f} | {interval} |') + for shape, row in pair['per_shape'].items(): + writer.writerow([pair['learned'], pair['dataset'], shape, row['queries'], + row['sort']['mrr'] * 100, row['expected']['mrr'] * 100]) + (output / 'adapter-effects.md').write_text('\n'.join(lines) + '\n') + groups = defaultdict(dict) + for detail in details: + if detail['graph_ablation']: + group = groups[detail['graph_ablation'], detail['calibration'], detail['answer_filter']] + if detail['inference_graph'] in group: + raise ValueError('Duplicate graph-ablation condition') + group[detail['inference_graph']] = detail + complete = [group for group in groups.values() if set(group) == {'train', 'train+valid'}] + for group in complete: + if not group['train']['graph_recipe_sha256'] or group['train']['graph_recipe_sha256'] != group['train+valid']['graph_recipe_sha256']: + raise ValueError('Graph ablation changed model, adapter, operators, beam, or query settings') + effects = starmap(effect, [(_trace(group['train+valid']), _trace(group['train'])) for group in complete]) + graph_pairs = [dict(train=group['train']['id'], train_valid=group['train+valid']['id'], dataset=group['train']['dataset'], + calibration=group['train']['calibration'], answer_filter=group['train']['answer_filter'], **values) + for group, values in zip(complete, effects)] + write_json(output / 'graph-effects.json', graph_pairs) + lines = ['# Inference graph effect', '', + 'Train+valid minus train, MRR ×100. Checkpoints, operators, queries and answer-filter policy are fixed.', '', + '| Train entry | Dataset | Sort Δ | Expected random Δ | Expected random 95% CI |', + '|---|---|---:|---:|---|'] + with (output / 'graph-effects.csv').open('w', newline='') as stream: + writer = csv.writer(stream) + writer.writerow(['train_entry', 'train_valid_entry', 'dataset', 'type', 'queries', 'sort_delta_mrr', 'expected_delta_mrr']) + for pair in graph_pairs: + ci = pair['macro']['expected']['mrr_ci95'] + interval = f'[{ci[0] * 100:+.2f}, {ci[1] * 100:+.2f}]' if ci is not None else '—' + lines.append(f'| {pair["train"]} | {pair["dataset"]} | {pair["macro"]["sort"]["mrr"] * 100:+.2f} | ' + f'{pair["macro"]["expected"]["mrr"] * 100:+.2f} | {interval} |') + for shape, row in pair['per_shape'].items(): + writer.writerow([pair['train'], pair['train_valid'], pair['dataset'], shape, row['queries'], + row['sort']['mrr'] * 100, row['expected']['mrr'] * 100]) + (output / 'graph-effects.md').write_text('\n'.join(lines) + '\n') + released = [control for control in details if control['filter_paired_with'] in lookup] + if any((lookup[control['filter_paired_with']]['answer_filter'], control['answer_filter']) != ('corrected', 'released') + for control in released): + raise ValueError('Invalid corrected/released filter comparison') + effects = starmap(effect, [(_trace(lookup[control['filter_paired_with']]), _trace(control)) for control in released]) + filter_pairs = [dict(corrected=control['filter_paired_with'], released=control['id'], dataset=control['dataset'], **values) + for control, values in zip(released, effects)] + write_json(output / 'filter-effects.json', filter_pairs) + lines = ['# Answer-filter effect', '', + 'Corrected minus released filters, MRR ×100. Predictions and released hard targets are identical.', '', + '| Entry | Dataset | Sort Δ | Expected random Δ | Expected random 95% CI |', + '|---|---|---:|---:|---|'] + with (output / 'filter-effects.csv').open('w', newline='') as stream: + writer = csv.writer(stream) + writer.writerow(['entry', 'dataset', 'type', 'queries', 'sort_delta_mrr', 'expected_delta_mrr']) + for pair in filter_pairs: + ci = pair['macro']['expected']['mrr_ci95'] + interval = f'[{ci[0] * 100:+.2f}, {ci[1] * 100:+.2f}]' if ci is not None else '—' + lines.append(f'| {pair["corrected"]} | {pair["dataset"]} | {pair["macro"]["sort"]["mrr"] * 100:+.2f} | ' + f'{pair["macro"]["expected"]["mrr"] * 100:+.2f} | {interval} |') + for shape, row in pair['per_shape'].items(): + writer.writerow([pair['corrected'], pair['dataset'], shape, row['queries'], + row['sort']['mrr'] * 100, row['expected']['mrr'] * 100]) + (output / 'filter-effects.md').write_text('\n'.join(lines) + '\n') + records = [dict(id=row['id'], dataset=row['dataset'], method=row['method'], calibration=detail['calibration'], + paired_with=detail['paired_with'], complete=row['complete_test'], answer_filter=detail['answer_filter'], + trace=_trace(detail)) + for row, detail in zip(rows, details)] + effects = suite_effects(records, bootstrap_samples=bootstrap_samples, seed=seed, starmap=starmap) + write_json(output / 'suite-effects.json', effects) + lines = ['# Suite-level paired effects', '', + 'Treatment minus baseline, MRR ×100, equal weight per query type within a dataset and per dataset within ' + 'a group. Adapter seeds are averaged per query before differencing; intervals resample queries within ' + 'dataset × type and are conditional on the frozen models, adapters and seeds.', '', + '| Comparison | Recipe | Suite | Group | Seeds | Sort Δ | Sort 95% CI | Expected Δ | Seed range (sort) |', + '|---|---|---|---|---:|---:|---|---:|---|'] + for effect in effects: + for group, values in effect['macro']['sort'].items(): + ci = values['ci95'] + interval = f'[{ci[0]:+.2f}, {ci[1]:+.2f}]' if ci is not None else '—' + seeds = [s['sort'][group] for s in effect['per_seed'] if group in s['sort']] + spread = f'{min(seeds):+.2f} to {max(seeds):+.2f}' if len(seeds) > 1 else '—' + lines.append(f'| {effect["comparison"]} | {effect["recipe"]} | {effect["suite"]} | {group} | {len(effect["per_seed"])} | ' + f'{values["delta"]:+.2f} | {interval} | {effect["macro"]["expected"][group]["delta"]:+.2f} | {spread} |') + (output / 'suite-effects.md').write_text('\n'.join(lines) + '\n') + return rows + + +def query_scores(path): + """{query_id: (shape, hard answers, sort MRR, expected MRR)} of one rank trace.""" + return {q['query_id']: (q['shape'], q['hard_answers'], q['sort']['mrr'], q['expected']['mrr']) for q in trace_queries(path)} + + +def paired_strata(treatments, baseline): + """{(policy, shape): per-query MRR differences}, treatments averaged per query (seed means). + + Every trace must cover the same queries with the same types and hard-answer counts. + """ + return dataset_strata(treatments, baseline)[0] + + +def dataset_strata(treatments, baseline, singles=False): + """paired_strata of the treatments and, with ``singles``, of each treatment alone; every trace is read once.""" + reference = query_scores(baseline) + scores = [query_scores(path) for path in treatments] + return _strata(scores, reference, baseline), [_strata([s], reference, baseline) for s in scores] if singles else [] + + +def _strata(scores, reference, baseline): + if any(set(s) != set(reference) for s in scores) or not reference: + raise ValueError(f'Suite comparison needs identical nonempty query coverage: {baseline}') + strata = defaultdict(list) + for query_id, (shape, answers, sort_mrr, expected_mrr) in reference.items(): + rows = [s[query_id] for s in scores] + if any(row[0] != shape or row[1] != answers for row in rows): + raise ValueError(f'Suite comparison changes query types or answers: {baseline}') + strata['sort', shape].append(100 * (sum(row[2] for row in rows) / len(rows) - sort_mrr)) + strata['expected', shape].append(100 * (sum(row[3] for row in rows) / len(rows) - expected_mrr)) + return {key: np.asarray(values) for key, values in strata.items()} + + +def macro_effects(by_dataset, groups, *, bootstrap_samples=2000, seed=0): + """{policy: {group: delta, ci95, datasets}} with equal type and dataset weights. + + by_dataset maps a dataset to its paired strata; groups map a name to (datasets, type filter). + The bootstrap resamples queries within each dataset × type stratum, in bounded memory. + """ + rng = np.random.default_rng(seed) + output = {} + for policy in ('sort', 'expected'): + output[policy] = {} + for group, (datasets, keep) in groups.items(): + members = [d for d in datasets if d in by_dataset] + shapes = {d: [s for p, s in by_dataset[d] if p == policy and keep(s)] for d in members} + members = [d for d in members if shapes[d]] + if not members: + continue + point = float(np.mean([np.mean([by_dataset[d][policy, s].mean() for s in shapes[d]]) for d in members])) + interval = None + if bootstrap_samples: + draws = np.zeros(bootstrap_samples) + for d in members: + for s in shapes[d]: + values = by_dataset[d][policy, s] + chunk = max(1, 4_000_000 // len(values)) + means = np.concatenate([values[rng.integers(0, len(values), size=(min(chunk, bootstrap_samples - start), len(values)))].mean(1) + for start in range(0, bootstrap_samples, chunk)]) + draws += means / len(shapes[d]) / len(members) + interval = [float(v) for v in np.quantile(draws, [.025, .975])] + output[policy][group] = dict(delta=point, ci95=interval, datasets=len(members)) + return output + + +NEGATION = frozenset(s for s in PLUS_H_SHAPES if 'n' in s) + + +def any_type(shape): + return True + + +def epfo_type(shape): + return shape not in NEGATION + + +def negation_type(shape): + return shape in NEGATION + + +# A run ID is method and recipe, an optional -seedN tag of the adapter training seed, the dataset, and the suffixes +# of paired controls and evaluation conditions. +SEED = re.compile(r'-seed(\d+)(?=-|$)') +CONDITION_SUFFIXES = ('-graph-train-valid', '-graph-train', '-released-filters') + + +def identity_run(record): + """A no-adapter control: paired with a learned run or identity-calibrated.""" + return bool(record.get('paired_with') or record.get('calibration') == 'without-adapter' + or '-without-adapter' in (record.get('id') or '')) + + +def system_of(record): + """((method, identity, recipe), seed tag) of one result; recipes exclude dataset, seed and condition.""" + text = record['id'].replace(record['dataset'], '').replace('-without-adapter', '') + for suffix in CONDITION_SUFFIXES: + text = text.replace(suffix, '') + match = SEED.search(text) + recipe = re.sub('-+', '-', SEED.sub('', text)).strip('-') or record['method'] + return (record['method'], identity_run(record), recipe), int(match.group(1)) if match else 0 + + +def suite_effects(records, *, bootstrap_samples=2000, seed=0, starmap=None): + """Suite-level paired effects of every adapter recipe: against its no-adapter control and against UltraQuery. + + Seed replicates (entry IDs differing only by -seedN) are averaged per query; the control, + which identity calibration makes independent of the adapter seed, is its lowest seed tag. + Only complete test runs with corrected (+H) or released (UltraQuery) filters are compared. + ``starmap`` (see ``parallel``) computes the per-dataset strata and the bootstraps. + """ + from dicee.query_answering.catalog import BENCHMARK_DATASETS, PLUS_H_DATASETS, dataset_spec + + starmap = starmap or (lambda function, calls: [function(*arguments) for arguments in calls]) + suites = {'ultraquery': [d for d in BENCHMARK_DATASETS if dataset_spec(d)[0] in ('transductive', 'inductive-e', 'inductive-er')], + 'plus_h': list(PLUS_H_DATASETS)} + systems = defaultdict(lambda: defaultdict(dict)) + for record in records: + target_filter = 'corrected' if record['dataset'] in PLUS_H_DATASETS else 'released' + if not record['complete'] or record['answer_filter'] != target_filter or record['id'].endswith('-released-filters'): + continue + system, tag = system_of(record) + systems[system][record['dataset']][tag] = record['trace'] + plans = [] + for (method, identity, recipe), runs in sorted(systems.items()): + if identity or not method.endswith('-adapter'): + continue + control = systems.get((method, True, recipe), {}) + baseline = systems.get(('ultraquery', False, 'ultraquery'), {}) + for suite, datasets in suites.items(): + present = [d for d in datasets if d in runs] + if not present: + continue + families = {'all': (datasets, any_type), 'epfo': (datasets, epfo_type), 'negation': (datasets, negation_type)} + if suite == 'ultraquery': + families |= {family: ([d for d in datasets if dataset_spec(d)[0] == family], any_type) + for family in ('transductive', 'inductive-e', 'inductive-er')} + seeds = sorted(set.intersection(*(set(runs[d]) for d in present))) + for comparison, reference in (('adapter-minus-identity', control), ('adapter-minus-ultraquery', baseline), + ('identity-minus-ultraquery', baseline if control else {})): + if comparison == 'identity-minus-ultraquery': + treatments = {d: [control[d][min(control[d])]] for d in present if d in control} + else: + treatments = {d: [runs[d][tag] for tag in seeds] for d in present} + shared = [d for d in present if d in reference and d in treatments and treatments[d]] + if shared: + plans.append(dict(comparison=comparison, method=method, recipe=recipe, suite=suite, datasets=datasets, + families=families, seeds=seeds, shared=shared, treatments=treatments, + base={d: reference[d][min(reference[d])] for d in shared})) + # Read every dataset's traces once, for all comparisons at the same time. + calls = [(plan['treatments'][d], plan['base'][d], plan['comparison'] != 'identity-minus-ultraquery') + for plan in plans for d in plan['shared']] + computed = iter(starmap(dataset_strata, calls)) + for plan in plans: + plan['strata'] = {d: next(computed) for d in plan['shared']} + macros = starmap(partial(macro_effects, bootstrap_samples=bootstrap_samples, seed=seed), + [({d: pooled for d, (pooled, _) in plan['strata'].items()}, plan['families']) for plan in plans]) + output = [] + for plan, macro in zip(plans, macros): + per_seed = [] + for index, tag in enumerate(plan['seeds'] if plan['comparison'] != 'identity-minus-ultraquery' else []): + values = macro_effects({d: singles[index] for d, (_, singles) in plan['strata'].items()}, plan['families'], bootstrap_samples=0) + per_seed.append(dict(seed_tag=tag, **{p: {g: v['delta'] for g, v in values[p].items()} for p in values})) + output.append(dict(comparison=plan['comparison'], method=plan['method'], recipe=plan['recipe'], suite=plan['suite'], + datasets=len(plan['shared']), complete_suite=len(plan['shared']) == len(plan['datasets']), + seed_tags=plan['seeds'] if plan['comparison'] != 'identity-minus-ultraquery' else [], + macro=macro, per_seed=per_seed, + uncertainty=dict(unit='paired query', stratification='dataset x query type', + resamples=bootstrap_samples, seed=seed, + scope='Conditional on frozen models, adapters and the evaluated seeds.'))) + return output diff --git a/benchmarks/cqa/requirements.lock b/benchmarks/cqa/requirements.lock new file mode 100644 index 000000000..999dafc9f --- /dev/null +++ b/benchmarks/cqa/requirements.lock @@ -0,0 +1,68 @@ +# Python 3.11 / Linux; exact installed versions and approved distribution hashes. +aiohappyeyeballs==2.6.1 --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +aiohttp==3.13.3 --hash=sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf --hash=sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c --hash=sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c --hash=sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423 --hash=sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f --hash=sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40 --hash=sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2 --hash=sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf --hash=sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821 --hash=sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64 --hash=sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7 --hash=sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998 --hash=sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d --hash=sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea --hash=sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463 --hash=sha256:2ba0eea45eb5cc3172dbfc497c066f19c41bac70963ea1a67d51fc92e4cf9a80 --hash=sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4 --hash=sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767 --hash=sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43 --hash=sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592 --hash=sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a --hash=sha256:31a83ea4aead760dfcb6962efb1d861db48c34379f2ff72db9ddddd4cda9ea2e --hash=sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687 --hash=sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8 --hash=sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261 --hash=sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd --hash=sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a --hash=sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4 --hash=sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587 --hash=sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91 --hash=sha256:40c5e40ecc29ba010656c18052b877a1c28f84344825efa106705e835c28530f --hash=sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3 --hash=sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344 --hash=sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6 --hash=sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3 --hash=sha256:4ae5b5a0e1926e504c81c5b84353e7a5516d8778fbbff00429fe7b05bb25cbce --hash=sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808 --hash=sha256:56339a36b9f1fc708260c76c87e593e2afb30d26de9ae1eb445b5e051b98a7a1 --hash=sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29 --hash=sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3 --hash=sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b --hash=sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51 --hash=sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c --hash=sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926 --hash=sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64 --hash=sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f --hash=sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b --hash=sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e --hash=sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440 --hash=sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6 --hash=sha256:69c56fbc1993fa17043e24a546959c0178fe2b5782405ad4559e6c13975c15e3 --hash=sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d --hash=sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415 --hash=sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279 --hash=sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce --hash=sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603 --hash=sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0 --hash=sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c --hash=sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf --hash=sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591 --hash=sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540 --hash=sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e --hash=sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26 --hash=sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a --hash=sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845 --hash=sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a --hash=sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9 --hash=sha256:8a60e60746623925eab7d25823329941aee7242d559baa119ca2b253c88a7bd6 --hash=sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba --hash=sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df --hash=sha256:947c26539750deeaee933b000fb6517cc770bbd064bad6033f1cff4803881e43 --hash=sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679 --hash=sha256:988a8c5e317544fdf0d39871559e67b6341065b87fceac641108c2096d5506b7 --hash=sha256:9a9dc347e5a3dc7dfdbc1f82da0ef29e388ddb2ed281bfce9dd8248a313e62b7 --hash=sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc --hash=sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29 --hash=sha256:9b174f267b5cfb9a7dba9ee6859cecd234e9a681841eb85068059bc867fb8f02 --hash=sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984 --hash=sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1 --hash=sha256:9ebf57d09e131f5323464bd347135a88622d1c0976e88ce15b670e7ad57e4bd6 --hash=sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632 --hash=sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56 --hash=sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239 --hash=sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168 --hash=sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88 --hash=sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc --hash=sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11 --hash=sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046 --hash=sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0 --hash=sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3 --hash=sha256:b46020d11d23fe16551466c77823df9cc2f2c1e63cc965daf67fa5eec6ca1877 --hash=sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1 --hash=sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c --hash=sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25 --hash=sha256:b99281b0704c103d4e11e72a76f1b543d4946fea7dd10767e7e1b5f00d4e5704 --hash=sha256:bae5c2ed2eae26cc382020edad80d01f36cb8e746da40b292e68fec40421dc6a --hash=sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033 --hash=sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1 --hash=sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29 --hash=sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d --hash=sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160 --hash=sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d --hash=sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f --hash=sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f --hash=sha256:c6b8568a3bb5819a0ad087f16d40e5a3fb6099f39ea1d5625a3edc1e923fc538 --hash=sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29 --hash=sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7 --hash=sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72 --hash=sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af --hash=sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455 --hash=sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57 --hash=sha256:e50a2e1404f063427c9d027378472316201a2290959a295169bcf25992d04558 --hash=sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c --hash=sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808 --hash=sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7 --hash=sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0 --hash=sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3 --hash=sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730 --hash=sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa --hash=sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940 +aiosignal==1.4.0 --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 +attrs==25.4.0 --hash=sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11 --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 +certifi==2026.1.4 --hash=sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c --hash=sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120 +charset-normalizer==3.4.4 --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 +filelock==3.20.2 --hash=sha256:a2241ff4ddde2a7cebddf78e39832509cb045d18ec1a09d7248d6bfc6bfbbe64 --hash=sha256:fbba7237d6ea277175a32c54bb71ef814a8546d8601269e1bfc388de333974e8 +frozenlist==1.8.0 --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd +fsspec==2025.12.0 --hash=sha256:8bf1fe301b7d8acfa6e8571e3b1c3d158f909666642431cc78a1b7b4dbc5ec5b --hash=sha256:c505de011584597b1060ff778bb664c1bc022e87921b0e4f10cc9c44f9635973 +idna==3.11 --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 +iniconfig==2.3.0 --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 +jinja2==3.1.6 --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +joblib==1.5.3 --hash=sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713 --hash=sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3 +lightning==2.6.0 --hash=sha256:881841716b59c1837ae0c562c2e64fea9bcf49ef9de3867bd1f868557ec23d04 --hash=sha256:f1a13a48909960a3454518486f113fae4fadb2db0e28e9c50d8d38d46c9dc3d6 +lightning-utilities==0.15.2 --hash=sha256:ad3ab1703775044bbf880dbf7ddaaac899396c96315f3aa1779cec9d618a9841 --hash=sha256:cdf12f530214a63dacefd713f180d1ecf5d165338101617b4742e8f22c032e24 +markupsafe==3.0.3 --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 +mpmath==1.3.0 --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c +multidict==6.7.0 --hash=sha256:03ca744319864e92721195fa28c7a3b2bc7b686246b35e4078c1e4d0eb5466d3 --hash=sha256:040f393368e63fb0f3330e70c26bfd336656bed925e5cbe17c9da839a6ab13ec --hash=sha256:05047ada7a2fde2631a0ed706f1fd68b169a681dfe5e4cf0f8e4cb6618bbc2cd --hash=sha256:0591b48acf279821a579282444814a2d8d0af624ae0bc600aa4d1b920b6e924b --hash=sha256:07f5594ac6d084cbb5de2df218d78baf55ef150b91f0ff8a21cc7a2e3a5a58eb --hash=sha256:08325c9e5367aa379a3496aa9a022fe8837ff22e00b94db256d3a1378c76ab32 --hash=sha256:08d4379f9744d8f78d98c8673c06e202ffa88296f009c71bbafe8a6bf847d01f --hash=sha256:0934f3843a1860dd465d38895c17fce1f1cb37295149ab05cd1b9a03afacb2a7 --hash=sha256:096f52730c3fb8ed419db2d44391932b63891b2c5ed14850a7e215c0ba9ade36 --hash=sha256:09929cab6fcb68122776d575e03c6cc64ee0b8fca48d17e135474b042ce515cd --hash=sha256:0a13fb8e748dfc94749f622de065dd5c1def7e0d2216dba72b1d8069a389c6ff --hash=sha256:0db4956f82723cc1c270de9c6e799b4c341d327762ec78ef82bb962f79cc07d8 --hash=sha256:123e2a72e20537add2f33a79e605f6191fba2afda4cbb876e35c1a7074298a7d --hash=sha256:14c9e076eede3b54c636f8ce1c9c252b5f057c62131211f0ceeec273810c9721 --hash=sha256:171b73bd4ee683d307599b66793ac80981b06f069b62eea1c9e29c9241aa66b0 --hash=sha256:18706cc31dbf402a7945916dd5cddf160251b6dab8a2c5f3d6d5a55949f676b3 --hash=sha256:19a1d55338ec1be74ef62440ca9e04a2f001a04d0cc49a4983dc320ff0f3212d --hash=sha256:2049be98fb57a31b4ccf870bf377af2504d4ae35646a19037ec271e4c07998aa --hash=sha256:2090d3718829d1e484706a2f525e50c892237b2bf9b17a79b059cb98cddc2f10 --hash=sha256:2397ab4daaf2698eb51a76721e98db21ce4f52339e535725de03ea962b5a3202 --hash=sha256:23bfeee5316266e5ee2d625df2d2c602b829435fc3a235c2ba2131495706e4a0 --hash=sha256:27e0b36c2d388dc7b6ced3406671b401e84ad7eb0656b8f3a2f46ed0ce483718 --hash=sha256:28b37063541b897fd6a318007373930a75ca6d6ac7c940dbe14731ffdd8d498e --hash=sha256:295a92a76188917c7f99cda95858c822f9e4aae5824246bba9b6b44004ddd0a6 --hash=sha256:29fe6740ebccba4175af1b9b87bf553e9c15cd5868ee967e010efcf94e4fd0f1 --hash=sha256:2a7baa46a22e77f0988e3b23d4ede5513ebec1929e34ee9495be535662c0dfe2 --hash=sha256:2d2cfeec3f6f45651b3d408c4acec0ebf3daa9bc8a112a084206f5db5d05b754 --hash=sha256:2f67396ec0310764b9222a1728ced1ab638f61aadc6226f17a71dd9324f9a99c --hash=sha256:30d193c6cc6d559db42b6bcec8a5d395d34d60c9877a0b71ecd7c204fcf15390 --hash=sha256:31bae522710064b5cbeddaf2e9f32b1abab70ac6ac91d42572502299e9953128 --hash=sha256:329aa225b085b6f004a4955271a7ba9f1087e39dcb7e65f6284a988264a63912 --hash=sha256:363eb68a0a59bd2303216d2346e6c441ba10d36d1f9969fcb6f1ba700de7bb5c --hash=sha256:394fc5c42a333c9ffc3e421a4c85e08580d990e08b99f6bf35b4132114c5dcb3 --hash=sha256:3996b50c3237c4aec17459217c1e7bbdead9a22a0fcd3c365564fbd16439dde6 --hash=sha256:39f1719f57adbb767ef592a50ae5ebb794220d1188f9ca93de471336401c34d2 --hash=sha256:3b29b980d0ddbecb736735ee5bef69bb2ddca56eff603c86f3f29a1128299b4f --hash=sha256:3ba3ef510467abb0667421a286dc906e30eb08569365f5cdb131d7aff7c2dd84 --hash=sha256:3bab1e4aff7adaa34410f93b1f8e57c4b36b9af0426a76003f441ee1d3c7e842 --hash=sha256:3d7b6ccce016e29df4b7ca819659f516f0bc7a4b3efa3bb2012ba06431b044f9 --hash=sha256:3da4fb467498df97e986af166b12d01f05d2e04f978a9c1c680ea1988e0bc4b6 --hash=sha256:3e56d780c238f9e1ae66a22d2adf8d16f485381878250db8d496623cd38b22bd --hash=sha256:3e8bfdd0e487acf992407a140d2589fe598238eaeffa3da8448d63a63cd363f8 --hash=sha256:44b546bd3eb645fd26fb949e43c02a25a2e632e2ca21a35e2e132c8105dc8599 --hash=sha256:478cc36476687bac1514d651cbbaa94b86b0732fb6855c60c673794c7dd2da62 --hash=sha256:490dab541a6a642ce1a9d61a4781656b346a55c13038f0b1244653828e3a83ec --hash=sha256:4a0df7ff02397bb63e2fd22af2c87dfa39e8c7f12947bc524dbdc528282c7e34 --hash=sha256:4b73189894398d59131a66ff157837b1fafea9974be486d036bb3d32331fdbf0 --hash=sha256:4b7a9db5a870f780220e931d0002bbfd88fb53aceb6293251e2c839415c1b20e --hash=sha256:4c09703000a9d0fa3c3404b27041e574cc7f4df4c6563873246d0e11812a94b6 --hash=sha256:4d409aa42a94c0b3fa617708ef5276dfe81012ba6753a0370fcc9d0195d0a1fc --hash=sha256:4d72a9a2d885f5c208b0cb91ff2ed43636bb7e345ec839ff64708e04f69a13cc --hash=sha256:4ef089f985b8c194d341eb2c24ae6e7408c9a0e2e5658699c92f497437d88c3c --hash=sha256:51cb455de290ae462593e5b1cb1118c5c22ea7f0d3620d9940bf695cea5a4bd7 --hash=sha256:521f33e377ff64b96c4c556b81c55d0cfffb96a11c194fd0c3f1e56f3d8dd5a4 --hash=sha256:53a42d364f323275126aff81fb67c5ca1b7a04fda0546245730a55c8c5f24bc4 --hash=sha256:5aa873cbc8e593d361ae65c68f85faadd755c3295ea2c12040ee146802f23b38 --hash=sha256:654030da3197d927f05a536a66186070e98765aa5142794c9904555d3a9d8fb5 --hash=sha256:661709cdcd919a2ece2234f9bae7174e5220c80b034585d7d8a755632d3e2111 --hash=sha256:680878b9f3d45c31e1f730eef731f9b0bc1da456155688c6745ee84eb818e90e --hash=sha256:6843b28b0364dc605f21481c90fadb5f60d9123b442eb8a726bb74feef588a84 --hash=sha256:68af405971779d8b37198726f2b6fe3955db846fee42db7a4286fc542203934c --hash=sha256:6b4c3d199f953acd5b446bf7c0de1fe25d94e09e79086f8dc2f48a11a129cdf1 --hash=sha256:6bdce131e14b04fd34a809b6380dbfd826065c3e2fe8a50dbae659fa0c390546 --hash=sha256:716133f7d1d946a4e1b91b1756b23c088881e70ff180c24e864c26192ad7534a --hash=sha256:749a72584761531d2b9467cfbdfd29487ee21124c304c4b6cb760d8777b27f9c --hash=sha256:7516c579652f6a6be0e266aec0acd0db80829ca305c3d771ed898538804c2036 --hash=sha256:79dcf9e477bc65414ebfea98ffd013cb39552b5ecd62908752e0e413d6d06e38 --hash=sha256:7a0222514e8e4c514660e182d5156a415c13ef0aabbd71682fc714e327b95e99 --hash=sha256:7b022717c748dd1992a83e219587aabe45980d88969f01b316e78683e6285f64 --hash=sha256:7bf77f54997a9166a2f5675d1201520586439424c2511723a7312bdb4bcc034e --hash=sha256:7e73299c99939f089dd9b2120a04a516b95cdf8c1cd2b18c53ebf0de80b1f18f --hash=sha256:7ef6b61cad77091056ce0e7ce69814ef72afacb150b7ac6a3e9470def2198159 --hash=sha256:7f5170993a0dd3ab871c74f45c0a21a4e2c37a2f2b01b5f722a2ad9c6650469e --hash=sha256:803d685de7be4303b5a657b76e2f6d1240e7e0a8aa2968ad5811fa2285553a12 --hash=sha256:8891681594162635948a636c9fe0ff21746aeb3dd5463f6e25d9bea3a8a39ca1 --hash=sha256:8a19cdb57cd3df4cd865849d93ee14920fb97224300c88501f16ecfa2604b4e0 --hash=sha256:8a3862568a36d26e650a19bb5cbbba14b71789032aebc0423f8cc5f150730184 --hash=sha256:8b55d5497b51afdfde55925e04a022f1de14d4f4f25cdfd4f5d9b0aa96166851 --hash=sha256:8cfc12a8630a29d601f48d47787bd7eb730e475e83edb5d6c5084317463373eb --hash=sha256:9281bf5b34f59afbc6b1e477a372e9526b66ca446f4bf62592839c195a718b32 --hash=sha256:92abb658ef2d7ef22ac9f8bb88e8b6c3e571671534e029359b6d9e845923eb1b --hash=sha256:94218fcec4d72bc61df51c198d098ce2b378e0ccbac41ddbed5ef44092913288 --hash=sha256:95b5ffa4349df2887518bb839409bcf22caa72d82beec453216802f475b23c81 --hash=sha256:9600082733859f00d79dee64effc7aef1beb26adb297416a4ad2116fd61374bd --hash=sha256:960c60b5849b9b4f9dcc9bea6e3626143c252c74113df2c1540aebce70209b45 --hash=sha256:9b2fd74c52accced7e75de26023b7dccee62511a600e62311b918ec5c168fc2a --hash=sha256:9c0359b1ec12b1d6849c59f9d319610b7f20ef990a6d454ab151aa0e3b9f78ca --hash=sha256:9cf41880c991716f3c7cec48e2f19ae4045fc9db5fc9cff27347ada24d710bb5 --hash=sha256:9d14baca2ee12c1a64740d4531356ba50b82543017f3ad6de0deb943c5979abb --hash=sha256:9f474ad5acda359c8758c8accc22032c6abe6dc87a8be2440d097785e27a9349 --hash=sha256:9fb0211dfc3b51efea2f349ec92c114d7754dd62c01f81c3e32b765b70c45c9b --hash=sha256:9fe04da3f79387f450fd0061d4dd2e45a72749d31bf634aecc9e27f24fdc4b3f --hash=sha256:9ff96e8815eecacc6645da76c413eb3b3d34cfca256c70b16b286a687d013c32 --hash=sha256:a027ec240fe73a8d6281872690b988eed307cd7d91b23998ff35ff577ca688b5 --hash=sha256:a048ce45dcdaaf1defb76b2e684f997fb5abf74437b6cb7b22ddad934a964e34 --hash=sha256:a265acbb7bb33a3a2d626afbe756371dce0279e7b17f4f4eda406459c2b5ff1c --hash=sha256:a35c5fc61d4f51eb045061e7967cfe3123d622cd500e8868e7c0c592a09fedc4 --hash=sha256:a37bd74c3fa9d00be2d7b8eca074dc56bd8077ddd2917a839bd989612671ed17 --hash=sha256:a60a4d75718a5efa473ebd5ab685786ba0c67b8381f781d1be14da49f1a2dc60 --hash=sha256:a6ef16328011d3f468e7ebc326f24c1445f001ca1dec335b2f8e66bed3006394 --hash=sha256:a90af66facec4cebe4181b9e62a68be65e45ac9b52b67de9eec118701856e7ff --hash=sha256:ad9ce259f50abd98a1ca0aa6e490b58c316a0fce0617f609723e40804add2c00 --hash=sha256:afa8a2978ec65d2336305550535c9c4ff50ee527914328c8677b3973ade52b85 --hash=sha256:b15b3afff74f707b9275d5ba6a91ae8f6429c3ffb29bbfd216b0b375a56f13d7 --hash=sha256:b284e319754366c1aee2267a2036248b24eeb17ecd5dc16022095e747f2f4304 --hash=sha256:b2d7f80c4e1fd010b07cb26820aae86b7e73b681ee4889684fb8d2d4537aab13 --hash=sha256:b3bc26a951007b1057a1c543af845f1c7e3e71cc240ed1ace7bf4484aa99196e --hash=sha256:b3e34f3a1b8131ba06f1a73adab24f30934d148afcd5f5de9a73565a4404384e --hash=sha256:b4121773c49a0776461f4a904cdf6264c88e42218aaa8407e803ca8025872792 --hash=sha256:b61189b29081a20c7e4e0b49b44d5d44bb0dc92be3c6d06a11cc043f81bf9329 --hash=sha256:b6234e14f9314731ec45c42fc4554b88133ad53a09092cc48a88e771c125dadb --hash=sha256:b8512bac933afc3e45fb2b18da8e59b78d4f408399a960339598374d4ae3b56b --hash=sha256:ba672b26069957ee369cfa7fc180dde1fc6f176eaf1e6beaf61fbebbd3d9c000 --hash=sha256:bee7c0588aa0076ce77c0ea5d19a68d76ad81fcd9fe8501003b9a24f9d4000f6 --hash=sha256:c04a328260dfd5db8c39538f999f02779012268f54614902d0afc775d44e0a62 --hash=sha256:c1dcc7524066fa918c6a27d61444d4ee7900ec635779058571f70d042d86ed63 --hash=sha256:c6e99d9a65ca282e578dfea819cfa9c0a62b2499d8677392e09feaf305e9e6f5 --hash=sha256:ca43bdfa5d37bd6aee89d85e1d0831fb86e25541be7e9d376ead1b28974f8e5e --hash=sha256:caf53b15b1b7df9fbd0709aa01409000a2b4dd03a5f6f5cc548183c7c8f8b63c --hash=sha256:cc41db090ed742f32bd2d2c721861725e6109681eddf835d0a82bd3a5c382827 --hash=sha256:cd240939f71c64bd658f186330603aac1a9a81bf6273f523fca63673cb7378a8 --hash=sha256:ce8fdc2dca699f8dbf055a61d73eaa10482569ad20ee3c36ef9641f69afa8c91 --hash=sha256:d1bed1b467ef657f2a0ae62844a607909ef1c6889562de5e1d505f74457d0b96 --hash=sha256:d1d964afecdf3a8288789df2f5751dc0a8261138c3768d9af117ed384e538fad --hash=sha256:d4393e3581e84e5645506923816b9cc81f5609a778c7e7534054091acc64d1c6 --hash=sha256:d874eb056410ca05fed180b6642e680373688efafc7f077b2a2f61811e873a40 --hash=sha256:db99677b4457c7a5c5a949353e125ba72d62b35f74e26da141530fbb012218a7 --hash=sha256:dd32a49400a2c3d52088e120ee00c1e3576cbff7e10b98467962c74fdb762ed4 --hash=sha256:df0e3bf7993bdbeca5ac25aa859cf40d39019e015c9c91809ba7093967f7a648 --hash=sha256:e011555abada53f1578d63389610ac8a5400fc70ce71156b0aa30d326f1a5064 --hash=sha256:e2862408c99f84aa571ab462d25236ef9cb12a602ea959ba9c9009a54902fc73 --hash=sha256:e3aa16de190d29a0ea1b48253c57d99a68492c8dd8948638073ab9e74dc9410b --hash=sha256:e93a0617cd16998784bf4414c7e40f17a35d2350e5c6f0bd900d3a8e02bd3762 --hash=sha256:ea3334cabe4d41b7ccd01e4d349828678794edbc2d3ae97fc162a3312095092e --hash=sha256:eb866162ef2f45063acc7a53a88ef6fe8bf121d45c30ea3c9cd87ce7e191a8d4 --hash=sha256:ec81878ddf0e98817def1e77d4f50dae5ef5b0e4fe796fae3bd674304172416e --hash=sha256:efbb54e98446892590dc2458c19c10344ee9a883a79b5cec4bc34d6656e8d546 --hash=sha256:f0e77e3c0008bc9316e662624535b88d360c3a5d3f81e15cf12c139a75250046 --hash=sha256:f0feece2ef8ebc42ed9e2e8c78fc4aa3cf455733b507c09ef7406364c94376c6 --hash=sha256:f470f68adc395e0183b92a2f4689264d1ea4b40504a24d9882c27375e6662bb9 --hash=sha256:f844a1bbf1d207dd311a56f383f7eda2d0e134921d45751842d8235e7778965d --hash=sha256:f8a93b1c0ed2d04b97a5e9336fd2d33371b9a6e29ab7dd6503d63407c20ffbaf --hash=sha256:f8e5c0031b90ca9ce555e2e8fd5c3b02a25f14989cbc310701823832c99eb687 --hash=sha256:fb287618b9c7aa3bf8d825f02d9201b2f13078a5ed3b293c8f4d953917d84d5e --hash=sha256:fbafe31d191dfa7c4c51f7a6149c9fb7e914dcf9ffead27dcfd9f1ae382b3885 --hash=sha256:fbd18dc82d7bf274b37aa48d664534330af744e03bccf696d6f4c6042e7d19e7 +networkx==3.6.1 --hash=sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509 --hash=sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762 +numpy==1.26.4 --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f +nvidia-cublas-cu12==12.8.4.1 --hash=sha256:47e9b82132fa8d2b4944e708049229601448aaad7e6f296f630f2d1a32de35af --hash=sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142 --hash=sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0 +nvidia-cuda-cupti-cu12==12.8.90 --hash=sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed --hash=sha256:bb479dcdf7e6d4f8b0b01b115260399bf34154a1a2e9fe11c85c517d87efd98e --hash=sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182 +nvidia-cuda-nvrtc-cu12==12.8.93 --hash=sha256:7a4b6b2904850fe78e0bd179c4b655c404d4bb799ef03ddc60804247099ae909 --hash=sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994 --hash=sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8 +nvidia-cuda-runtime-cu12==12.8.90 --hash=sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d --hash=sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90 --hash=sha256:c0c6027f01505bfed6c3b21ec546f69c687689aad5f1a377554bc6ca4aa993a8 +nvidia-cudnn-cu12==9.10.2.21 --hash=sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8 --hash=sha256:c6288de7d63e6cf62988f0923f96dc339cea362decb1bf5b3141883392a7d65e --hash=sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8 +nvidia-cufft-cu12==11.3.3.83 --hash=sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74 --hash=sha256:7a64a98ef2a7c47f905aaf8931b69a3a43f27c55530c698bb2ed7c75c0b42cb7 --hash=sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a +nvidia-cufile-cu12==1.13.1.3 --hash=sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc --hash=sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a +nvidia-curand-cu12==10.3.9.90 --hash=sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9 --hash=sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd --hash=sha256:f149a8ca457277da854f89cf282d6ef43176861926c7ac85b2a0fbd237c587ec +nvidia-cusolver-cu12==11.7.3.90 --hash=sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450 --hash=sha256:4a550db115fcabc4d495eb7d39ac8b58d4ab5d8e63274d3754df1c0ad6a22d34 --hash=sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0 +nvidia-cusparse-cu12==12.5.8.93 --hash=sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b --hash=sha256:9a33604331cb2cac199f2e7f5104dfbb8a5a898c367a53dfda9ff2acb6b6b4dd --hash=sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc +nvidia-cusparselt-cu12==0.7.1 --hash=sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5 --hash=sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623 --hash=sha256:f67fbb5831940ec829c9117b7f33807db9f9678dc2a617fbe781cac17b4e1075 +nvidia-nccl-cu12==2.27.5 --hash=sha256:31432ad4d1fb1004eb0c56203dc9bc2178a1ba69d1d9e02d64a6938ab5e40e7a --hash=sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457 +nvidia-nvjitlink-cu12==12.8.93 --hash=sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88 --hash=sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7 --hash=sha256:bd93fbeeee850917903583587f4fc3a4eafa022e34572251368238ab5e6bd67f +nvidia-nvshmem-cu12==3.3.20 --hash=sha256:0b0b960da3842212758e4fa4696b94f129090b30e5122fea3c5345916545cff0 --hash=sha256:d00f26d3f9b2e3c3065be895e3059d6479ea5c638a3f38c9fec49b1b9dd7c1e5 +nvidia-nvtx-cu12==12.8.90 --hash=sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f --hash=sha256:619c8304aedc69f02ea82dd244541a83c3d9d40993381b3b590f1adaed3db41e --hash=sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615 +packaging==25.0 --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f +pandas==2.3.3 --hash=sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7 --hash=sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593 --hash=sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5 --hash=sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791 --hash=sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73 --hash=sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec --hash=sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4 --hash=sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5 --hash=sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac --hash=sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084 --hash=sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c --hash=sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87 --hash=sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35 --hash=sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250 --hash=sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c --hash=sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826 --hash=sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9 --hash=sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713 --hash=sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1 --hash=sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523 --hash=sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3 --hash=sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78 --hash=sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53 --hash=sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c --hash=sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21 --hash=sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5 --hash=sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff --hash=sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45 --hash=sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110 --hash=sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493 --hash=sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b --hash=sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450 --hash=sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86 --hash=sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8 --hash=sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98 --hash=sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89 --hash=sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66 --hash=sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b --hash=sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8 --hash=sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29 --hash=sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6 --hash=sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc --hash=sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2 --hash=sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788 --hash=sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa --hash=sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151 --hash=sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838 --hash=sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b --hash=sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a --hash=sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d --hash=sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908 --hash=sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0 --hash=sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b --hash=sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c --hash=sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee +pluggy==1.6.0 --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 +polars==1.36.1 --hash=sha256:12c7616a2305559144711ab73eaa18814f7aa898c522e7645014b68f1432d54c --hash=sha256:853c1bbb237add6a5f6d133c15094a9b727d66dd6a4eb91dbb07cdb056b2b8ef +polars-runtime-32==1.36.1 --hash=sha256:201c2cfd80ceb5d5cd7b63085b5fd08d6ae6554f922bcb941035e39638528a09 --hash=sha256:327b621ca82594f277751f7e23d4b939ebd1be18d54b4cdf7a2f8406cecc18b2 --hash=sha256:809e73857be71250141225ddd5d2b30c97e6340aeaa0d445f930e01bef6888dc --hash=sha256:899b9ad2e47ceb31eb157f27a09dbc2047efbf4969a923a6b1ba7f0412c3e64c --hash=sha256:ab0d1f23084afee2b97de8c37aa3e02ec3569749ae39571bd89e7a8b11ae9e83 --hash=sha256:cc17101f28c9a169ff8b5b8d4977a3683cd403621841623825525f440b564cf0 --hash=sha256:d9d077bb9df711bc635a86540df48242bb91975b353e53ef261c6fae6cb0948f +propcache==0.4.1 --hash=sha256:0002004213ee1f36cfb3f9a42b5066100c44276b9b72b4e1504cddd3d692e86e --hash=sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4 --hash=sha256:005f08e6a0529984491e37d8dbc3dd86f84bd78a8ceb5fa9a021f4c48d4984be --hash=sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3 --hash=sha256:05674a162469f31358c30bcaa8883cb7829fa3110bf9c0991fe27d7896c42d85 --hash=sha256:060b16ae65bc098da7f6d25bf359f1f31f688384858204fe5d652979e0015e5b --hash=sha256:120c964da3fdc75e3731aa392527136d4ad35868cc556fd09bb6d09172d9a367 --hash=sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf --hash=sha256:17612831fda0138059cc5546f4d12a2aacfb9e47068c06af35c400ba58ba7393 --hash=sha256:182b51b421f0501952d938dc0b0eb45246a5b5153c50d42b495ad5fb7517c888 --hash=sha256:1cdb7988c4e5ac7f6d175a28a9aa0c94cb6f2ebe52756a3c0cda98d2809a9e37 --hash=sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8 --hash=sha256:1f0978529a418ebd1f49dad413a2b68af33f85d5c5ca5c6ca2a3bed375a7ac60 --hash=sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1 --hash=sha256:296f4c8ed03ca7476813fe666c9ea97869a8d7aec972618671b33a38a5182ef4 --hash=sha256:2ad890caa1d928c7c2965b48f3a3815c853180831d0e5503d35cf00c472f4717 --hash=sha256:2b16ec437a8c8a965ecf95739448dd938b5c7f56e67ea009f4300d8df05f32b7 --hash=sha256:2bb07ffd7eaad486576430c89f9b215f9e4be68c4866a96e97db9e97fead85dc --hash=sha256:333ddb9031d2704a301ee3e506dc46b1fe5f294ec198ed6435ad5b6a085facfe --hash=sha256:357f5bb5c377a82e105e44bd3d52ba22b616f7b9773714bff93573988ef0a5fb --hash=sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75 --hash=sha256:364426a62660f3f699949ac8c621aad6977be7126c5807ce48c0aeb8e7333ea6 --hash=sha256:381914df18634f5494334d201e98245c0596067504b9372d8cf93f4bb23e025e --hash=sha256:3d233076ccf9e450c8b3bc6720af226b898ef5d051a2d145f7d765e6e9f9bcff --hash=sha256:3d902a36df4e5989763425a8ab9e98cd8ad5c52c823b34ee7ef307fd50582566 --hash=sha256:3f7124c9d820ba5548d431afb4632301acf965db49e666aa21c305cbe8c6de12 --hash=sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367 --hash=sha256:41a89040cb10bd345b3c1a873b2bf36413d48da1def52f268a055f7398514874 --hash=sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf --hash=sha256:473c61b39e1460d386479b9b2f337da492042447c9b685f28be4f74d3529e566 --hash=sha256:49a2dc67c154db2c1463013594c458881a069fcf98940e61a0569016a583020a --hash=sha256:4b536b39c5199b96fc6245eb5fb796c497381d3942f169e44e8e392b29c9ebcc --hash=sha256:4c3c70630930447f9ef1caac7728c8ad1c56bc5015338b20fed0d08ea2480b3a --hash=sha256:4d3df5fa7e36b3225954fba85589da77a0fe6a53e3976de39caf04a0db4c36f1 --hash=sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6 --hash=sha256:501d20b891688eb8e7aa903021f0b72d5a55db40ffaab27edefd1027caaafa61 --hash=sha256:521a463429ef54143092c11a77e04056dd00636f72e8c45b70aaa3140d639726 --hash=sha256:5558992a00dfd54ccbc64a32726a3357ec93825a418a401f5cc67df0ac5d9e49 --hash=sha256:55c72fd6ea2da4c318e74ffdf93c4fe4e926051133657459131a95c846d16d44 --hash=sha256:564d9f0d4d9509e1a870c920a89b2fec951b44bf5ba7d537a9e7c1ccec2c18af --hash=sha256:580e97762b950f993ae618e167e7be9256b8353c2dcd8b99ec100eb50f5286aa --hash=sha256:5a103c3eb905fcea0ab98be99c3a9a5ab2de60228aa5aceedc614c0281cf6153 --hash=sha256:5c3310452e0d31390da9035c348633b43d7e7feb2e37be252be6da45abd1abcc --hash=sha256:5d4e2366a9c7b837555cf02fb9be2e3167d333aff716332ef1b7c3a142ec40c5 --hash=sha256:5fd37c406dd6dc85aa743e214cef35dc54bbdd1419baac4f6ae5e5b1a2976938 --hash=sha256:60a8fda9644b7dfd5dece8c61d8a85e271cb958075bfc4e01083c148b61a7caf --hash=sha256:66c1f011f45a3b33d7bcb22daed4b29c0c9e2224758b6be00686731e1b46f925 --hash=sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8 --hash=sha256:678ae89ebc632c5c204c794f8dab2837c5f159aeb59e6ed0539500400577298c --hash=sha256:67fad6162281e80e882fb3ec355398cf72864a54069d060321f6cd0ade95fe85 --hash=sha256:6918ecbd897443087a3b7cd978d56546a812517dcaaca51b49526720571fa93e --hash=sha256:6f6ff873ed40292cd4969ef5310179afd5db59fdf055897e282485043fc80ad0 --hash=sha256:6f8b465489f927b0df505cbe26ffbeed4d6d8a2bbc61ce90eb074ff129ef0ab1 --hash=sha256:71b749281b816793678ae7f3d0d84bd36e694953822eaad408d682efc5ca18e0 --hash=sha256:74c1fb26515153e482e00177a1ad654721bf9207da8a494a0c05e797ad27b992 --hash=sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db --hash=sha256:824e908bce90fb2743bd6b59db36eb4f45cd350a39637c9f73b1c1ea66f5b75f --hash=sha256:8326e144341460402713f91df60ade3c999d601e7eb5ff8f6f7862d54de0610d --hash=sha256:8873eb4460fd55333ea49b7d189749ecf6e55bf85080f11b1c4530ed3034cba1 --hash=sha256:89eb3fa9524f7bec9de6e83cf3faed9d79bffa560672c118a96a171a6f55831e --hash=sha256:8c9b3cbe4584636d72ff556d9036e0c9317fa27b3ac1f0f558e7e84d1c9c5900 --hash=sha256:8e57061305815dfc910a3634dcf584f08168a8836e6999983569f51a8544cd89 --hash=sha256:929d7cbe1f01bb7baffb33dc14eb5691c95831450a26354cd210a8155170c93a --hash=sha256:92d1935ee1f8d7442da9c0c4fa7ac20d07e94064184811b685f5c4fada64553b --hash=sha256:948dab269721ae9a87fd16c514a0a2c2a1bdb23a9a61b969b0f9d9ee2968546f --hash=sha256:981333cb2f4c1896a12f4ab92a9cc8f09ea664e9b7dbdc4eff74627af3a11c0f --hash=sha256:990f6b3e2a27d683cb7602ed6c86f15ee6b43b1194736f9baaeb93d0016633b1 --hash=sha256:99d43339c83aaf4d32bda60928231848eee470c6bda8d02599cc4cebe872d183 --hash=sha256:9a0bd56e5b100aef69bd8562b74b46254e7c8812918d3baa700c8a8009b0af66 --hash=sha256:9a52009f2adffe195d0b605c25ec929d26b36ef986ba85244891dee3b294df21 --hash=sha256:9d2b6caef873b4f09e26ea7e33d65f42b944837563a47a94719cc3544319a0db --hash=sha256:9f302f4783709a78240ebc311b793f123328716a60911d667e0c036bc5dcbded --hash=sha256:a0ee98db9c5f80785b266eb805016e36058ac72c51a064040f2bc43b61101cdb --hash=sha256:a129e76735bc792794d5177069691c3217898b9f5cee2b2661471e52ffe13f19 --hash=sha256:a78372c932c90ee474559c5ddfffd718238e8673c340dc21fe45c5b8b54559a0 --hash=sha256:a9695397f85973bb40427dedddf70d8dc4a44b22f1650dd4af9eedf443d45165 --hash=sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778 --hash=sha256:ab2943be7c652f09638800905ee1bab2c544e537edb57d527997a24c13dc1455 --hash=sha256:ab4c29b49d560fe48b696cdcb127dd36e0bc2472548f3bf56cc5cb3da2b2984f --hash=sha256:af223b406d6d000830c6f65f1e6431783fc3f713ba3e6cc8c024d5ee96170a4b --hash=sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237 --hash=sha256:bcc9aaa5d80322bc2fb24bb7accb4a30f81e90ab8d6ba187aec0744bc302ad81 --hash=sha256:c07fda85708bc48578467e85099645167a955ba093be0a2dcba962195676e859 --hash=sha256:c0d4b719b7da33599dfe3b22d3db1ef789210a0597bc650b7cee9c77c2be8c5c --hash=sha256:c0ef0aaafc66fbd87842a3fe3902fd889825646bc21149eafe47be6072725835 --hash=sha256:c2b5e7db5328427c57c8e8831abda175421b709672f6cfc3d630c3b7e2146393 --hash=sha256:c30b53e7e6bda1d547cabb47c825f3843a0a1a42b0496087bb58d8fedf9f41b5 --hash=sha256:c80ee5802e3fb9ea37938e7eecc307fb984837091d5fd262bb37238b1ae97641 --hash=sha256:c9b822a577f560fbd9554812526831712c1436d2c046cedee4c3796d3543b144 --hash=sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74 --hash=sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db --hash=sha256:cbc3b6dfc728105b2a57c06791eb07a94229202ea75c59db644d7d496b698cac --hash=sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403 --hash=sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9 --hash=sha256:d472aeb4fbf9865e0c6d622d7f4d54a4e101a89715d8904282bb5f9a2f476c3f --hash=sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311 --hash=sha256:d82ad62b19645419fe79dd63b3f9253e15b30e955c0170e5cebc350c1844e581 --hash=sha256:d8f353eb14ee3441ee844ade4277d560cdd68288838673273b978e3d6d2c8f36 --hash=sha256:daede9cd44e0f8bdd9e6cc9a607fc81feb80fae7a5fc6cecaff0e0bb32e42d00 --hash=sha256:db65d2af507bbfbdcedb254a11149f894169d90488dd3e7190f7cdcb2d6cd57a --hash=sha256:dee69d7015dc235f526fe80a9c90d65eb0039103fe565776250881731f06349f --hash=sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2 --hash=sha256:e35b88984e7fa64aacecea39236cee32dd9bd8c55f57ba8a75cf2399553f9bd7 --hash=sha256:e53f3a38d3510c11953f3e6a33f205c6d1b001129f972805ca9b42fc308bc239 --hash=sha256:e9b0d8d0845bbc4cfcdcbcdbf5086886bc8157aa963c31c777ceff7846c77757 --hash=sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72 --hash=sha256:ecef2343af4cc68e05131e45024ba34f6095821988a9d0a02aa7c73fcc448aa9 --hash=sha256:ed5a841e8bb29a55fb8159ed526b26adc5bdd7e8bd7bf793ce647cb08656cdf4 --hash=sha256:ee17f18d2498f2673e432faaa71698032b0127ebf23ae5974eeaf806c279df24 --hash=sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207 --hash=sha256:f10207adf04d08bec185bae14d9606a1444715bc99180f9331c9c02093e1959e --hash=sha256:f1d2f90aeec838a52f1c1a32fe9a619fefd5e411721a9117fbf82aea638fe8a1 --hash=sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d --hash=sha256:f7ee0e597f495cf415bcbd3da3caa3bd7e816b74d0d52b8145954c5e6fd3ff37 --hash=sha256:f93243fdc5657247533273ac4f86ae106cc6445a0efacb9a1bfe982fcfefd90c --hash=sha256:f95393b4d66bfae908c3ca8d169d5f79cd65636ae15b5e7a4f6e67af675adb0e --hash=sha256:fc38cba02d1acba4e2869eef1a57a43dfbd3d49a59bf90dda7444ec2be6a5570 --hash=sha256:fd0858c20f078a32cf55f7e81473d96dcf3b93fd2ccdb3d40fdf54b8573df3af --hash=sha256:fd138803047fb4c062b1c1dd95462f5209456bfab55c734458f15d11da288f8f --hash=sha256:fd2dbc472da1f772a4dae4fa24be938a6c544671a912e30529984dd80400cd88 --hash=sha256:fd6f30fdcf9ae2a70abd34da54f18da086160e4d7d9251f81f3da0ff84fc5a48 --hash=sha256:fe49d0a85038f36ba9e3ffafa1103e61170b28e95b16622e11be0a0ea07c6781 +psutil==7.2.1 --hash=sha256:05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1 --hash=sha256:08a2f175e48a898c8eb8eace45ce01777f4785bc744c90aa2cc7f2fa5462a266 --hash=sha256:0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442 --hash=sha256:2ceae842a78d1603753561132d5ad1b2f8a7979cb0c283f5b52fb4e6e14b1a79 --hash=sha256:35630d5af80d5d0d49cfc4d64c1c13838baf6717a13effb35869a5919b854cdf --hash=sha256:3fce5f92c22b00cdefd1645aa58ab4877a01679e901555067b1bd77039aa589f --hash=sha256:494c513ccc53225ae23eec7fe6e1482f1b8a44674241b54561f755a898650679 --hash=sha256:5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8 --hash=sha256:81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49 --hash=sha256:923f8653416604e356073e6e0bccbe7c09990acef442def2f5640dd0faa9689f --hash=sha256:93f3f7b0bb07711b49626e7940d6fe52aa9940ad86e8f7e74842e73189712129 --hash=sha256:99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67 --hash=sha256:ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6 --hash=sha256:b1b0671619343aa71c20ff9767eced0483e4fc9e1f489d50923738caf6a03c17 --hash=sha256:b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42 --hash=sha256:ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d --hash=sha256:cfbe6b40ca48019a51827f20d830887b3107a74a79b01ceb8cc8de4ccb17b672 --hash=sha256:d34d2ca888208eea2b5c68186841336a7f5e0b990edec929be909353a202768a --hash=sha256:ea46c0d060491051d39f0d2cff4f98d5c72b288289f57a21556cc7d504db37fc --hash=sha256:f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3 --hash=sha256:f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8 +pyarrow==22.0.0 --hash=sha256:001ea83a58024818826a9e3f89bf9310a114f7e26dfe404a4c32686f97bd7901 --hash=sha256:00626d9dc0f5ef3a75fe63fd68b9c7c8302d2b5bbc7f74ecaedba83447a24f84 --hash=sha256:0c34fe18094686194f204a3b1787a27456897d8a2d62caf84b61e8dfbc0252ae --hash=sha256:12fe549c9b10ac98c91cf791d2945e878875d95508e1a5d14091a7aaa66d9cf8 --hash=sha256:1a812a5b727bc09c3d7ea072c4eebf657c2f7066155506ba31ebf4792f88f016 --hash=sha256:252be4a05f9d9185bb8c18e83764ebcfea7185076c07a7a662253af3a8c07941 --hash=sha256:334f900ff08ce0423407af97e6c26ad5d4e3b0763645559ece6fbf3747d6a8f5 --hash=sha256:35ad0f0378c9359b3f297299c3309778bb03b8612f987399a0333a560b43862d --hash=sha256:3d600dc583260d845c7d8a6db540339dd883081925da2bd1c5cb808f720b3cd9 --hash=sha256:3e294c5eadfb93d78b0763e859a0c16d4051fc1c5231ae8956d61cb0b5666f5a --hash=sha256:3e739edd001b04f654b166204fc7a9de896cf6007eaff33409ee9e50ceaff754 --hash=sha256:44729980b6c50a5f2bfcc2668d36c569ce17f8b17bccaf470c4313dcbbf13c9d --hash=sha256:44d2d26cda26d18f7af7db71453b7b783788322d756e81730acb98f24eb90ace --hash=sha256:4c19236ae2402a8663a2c8f21f1870a03cc57f0bef7e4b6eb3238cc82944de80 --hash=sha256:69763ab2445f632d90b504a815a2a033f74332997052b721002298ed6de40f2e --hash=sha256:6dda1ddac033d27421c20d7a7943eec60be44e0db4e079f33cc5af3b8280ccde --hash=sha256:6f9762274496c244d951c819348afbcf212714902742225f649cf02823a6a10f --hash=sha256:710624ab925dc2b05a6229d47f6f0dac1c1155e6ed559be7109f684eba048a48 --hash=sha256:7388ac685cab5b279a41dfe0a6ccd99e4dbf322edfb63e02fc0443bf24134e91 --hash=sha256:77718810bd3066158db1e95a63c160ad7ce08c6b0710bc656055033e39cdad88 --hash=sha256:7a820d8ae11facf32585507c11f04e3f38343c1e784c9b5a8b1da5c930547fe2 --hash=sha256:8382ad21458075c2e66a82a29d650f963ce51c7708c7c0ff313a8c206c4fd5e8 --hash=sha256:84378110dd9a6c06323b41b56e129c504d157d1a983ce8f5443761eb5256bafc --hash=sha256:854794239111d2b88b40b6ef92aa478024d1e5074f364033e73e21e3f76b25e0 --hash=sha256:92843c305330aa94a36e706c16209cd4df274693e777ca47112617db7d0ef3d7 --hash=sha256:9bddc2cade6561f6820d4cd73f99a0243532ad506bc510a75a5a65a522b2d74d --hash=sha256:a4893d31e5ef780b6edcaf63122df0f8d321088bb0dee4c8c06eccb1ca28d145 --hash=sha256:a9d9ffdc2ab696f6b15b4d1f7cec6658e1d788124418cb30030afbae31c64746 --hash=sha256:ac93252226cf288753d8b46280f4edf3433bf9508b6977f8dd8526b521a1bbb9 --hash=sha256:b41f37cabfe2463232684de44bad753d6be08a7a072f6a83447eeaf0e4d2a215 --hash=sha256:b883fe6fd85adad7932b3271c38ac289c65b7337c2c132e9569f9d3940620730 --hash=sha256:b9d71701ce97c95480fecb0039ec5bb889e75f110da72005743451339262f4ce --hash=sha256:ba95112d15fd4f1105fb2402c4eab9068f0554435e9b7085924bcfaac2cc306f --hash=sha256:bba208d9c7decf9961998edf5c65e3ea4355d5818dd6cd0f6809bec1afb951cc --hash=sha256:bd0d42297ace400d8febe55f13fdf46e86754842b860c978dfec16f081e5c653 --hash=sha256:bea79263d55c24a32b0d79c00a1c58bb2ee5f0757ed95656b01c0fb310c5af3d --hash=sha256:c064e28361c05d72eed8e744c9605cbd6d2bb7481a511c74071fd9b24bc65d7d --hash=sha256:c3200cb41cdbc65156e5f8c908d739b0dfed57e890329413da2748d1a2cd1a4e --hash=sha256:c6c791b09c57ed76a18b03f2631753a4960eefbbca80f846da8baefc6491fcfe --hash=sha256:c6ec3675d98915bf1ec8b3c7986422682f7232ea76cad276f4c8abd5b7319b70 --hash=sha256:ce20fe000754f477c8a9125543f1936ea5b8867c5406757c224d745ed033e691 --hash=sha256:cedb9dd9358e4ea1d9bce3665ce0797f6adf97ff142c8e25b46ba9cdd508e9b6 --hash=sha256:e0a15757fccb38c410947df156f9749ae4a3c89b2393741a50521f39a8cf202a --hash=sha256:e6e95176209257803a8b3d0394f21604e796dadb643d2f7ca21b66c9c0b30c9a --hash=sha256:e70ff90c64419709d38c8932ea9fe1cc98415c4f87ea8da81719e43f02534bc9 --hash=sha256:ec1a15968a9d80da01e1d30349b2b0d7cc91e96588ee324ce1b5228175043e95 --hash=sha256:ec5d40dd494882704fb876c16fa7261a69791e784ae34e6b5992e977bd2e238c --hash=sha256:f633074f36dbc33d5c05b5dc75371e5660f1dbf9c8b1d95669def05e5425989c --hash=sha256:f7fe3dbe871294ba70d789be16b6e7e52b418311e166e0e3cba9522f0f437fb1 --hash=sha256:f963ba8c3b0199f9d6b794c90ec77545e05eadc83973897a4523c9e8d84e9340 +pygments==2.19.2 --hash=sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887 --hash=sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b +pytest==9.0.2 --hash=sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b --hash=sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11 +python-dateutil==2.9.0.post0 --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 +pytorch-lightning==2.6.0 --hash=sha256:25b0d4f05e1f33b72be0920c34d0465777fe5f623228f9d6252b4b0f685d7037 --hash=sha256:ee72cff4b8c983ecfaae8599382544bd5236d9eb300adc7dd305f359195f4e79 +pytz==2025.2 --hash=sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3 --hash=sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00 +pyyaml==6.0.3 --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 +regex==2025.11.3 --hash=sha256:04d2765516395cf7dda331a244a3282c0f5ae96075f728629287dfa6f76ba70a --hash=sha256:087511f5c8b7dfbe3a03f5d5ad0c2a33861b1fc387f21f6f60825a44865a385a --hash=sha256:08b884f4226602ad40c5d55f52bf91a9df30f513864e0054bad40c0e9cf1afb7 --hash=sha256:0d31e08426ff4b5b650f68839f5af51a92a5b51abd8554a60c2fbc7c71f25d0b --hash=sha256:0f9397d561a4c16829d4e6ff75202c1c08b68a3bdbfe29dbfcdb31c9830907c6 --hash=sha256:10483eefbfb0adb18ee9474498c9a32fcf4e594fbca0543bb94c48bac6183e2e --hash=sha256:149eb0bba95231fb4f6d37c8f760ec9fa6fabf65bab555e128dde5f2475193ec --hash=sha256:1e00ec2970aab10dc5db34af535f21fcf32b4a31d99e34963419636e2f85ae39 --hash=sha256:1eb1ebf6822b756c723e09f5186473d93236c06c579d2cc0671a722d2ab14281 --hash=sha256:1fedc720f9bb2494ce31a58a1631f9c82df6a09b49c19517ea5cc280b4541e01 --hash=sha256:1ff0d190c7f68ae7769cd0313fe45820ba07ffebfddfaa89cc1eb70827ba0ddc --hash=sha256:2292cd5a90dab247f9abe892ac584cb24f0f54680c73fcb4a7493c66c2bf2467 --hash=sha256:22b29dda7e1f7062a52359fca6e58e548e28c6686f205e780b02ad8ef710de36 --hash=sha256:22c12d837298651e5550ac1d964e4ff57c3f56965fc1812c90c9fb2028eaf267 --hash=sha256:22dd622a402aad4558277305350699b2be14bc59f64d64ae1d928ce7d072dced --hash=sha256:22e7d1cdfa88ef33a2ae6aa0d707f9255eb286ffbd90045f1088246833223aee --hash=sha256:28ba4d69171fc6e9896337d4fc63a43660002b7da53fc15ac992abcf3410917c --hash=sha256:2ab815eb8a96379a27c3b6157fcb127c8f59c36f043c1678110cea492868f1d5 --hash=sha256:2b441a4ae2c8049106e8b39973bfbddfb25a179dda2bdb99b0eeb60c40a6a3af --hash=sha256:2fa2eed3f76677777345d2f81ee89f5de2f5745910e805f7af7386a920fa7313 --hash=sha256:32f74f35ff0f25a5021373ac61442edcb150731fbaa28286bbc8bb1582c89d02 --hash=sha256:3809988f0a8b8c9dcc0f92478d6501fac7200b9ec56aecf0ec21f4a2ec4b6009 --hash=sha256:3839967cf4dc4b985e1570fd8d91078f0c519f30491c60f9ac42a8db039be204 --hash=sha256:38af559ad934a7b35147716655d4a2f79fcef2d695ddfe06a06ba40ae631fa7e --hash=sha256:3a91e4a29938bc1a082cc28fdea44be420bf2bebe2665343029723892eb073e1 --hash=sha256:3b30bc921d50365775c09a7ed446359e5c0179e9e2512beec4a60cbcef6ddd50 --hash=sha256:3b3a5f320136873cc5561098dfab677eea139521cb9a9e8db98b7e64aef44cbc --hash=sha256:3bf28b1873a8af8bbb58c26cc56ea6e534d80053b41fb511a35795b6de507e6a --hash=sha256:3e0b11b2b2433d1c39c7c7a30e3f3d0aeeea44c2a8d0bae28f6b95f639927a69 --hash=sha256:3e816cc9aac1cd3cc9a4ec4d860f06d40f994b5c7b4d03b93345f44e08cc68bf --hash=sha256:3f8bf11a4827cc7ce5a53d4ef6cddd5ad25595d3c1435ef08f76825851343154 --hash=sha256:435bbad13e57eb5606a68443af62bed3556de2f46deb9f7d4237bc2f1c9fb3a0 --hash=sha256:43b4fb020e779ca81c1b5255015fe2b82816c76ec982354534ad9ec09ad7c9e3 --hash=sha256:442d86cf1cfe4faabf97db7d901ef58347efd004934da045c745e7b5bd57ac49 --hash=sha256:44f264d4bf02f3176467d90b294d59bf1db9fe53c141ff772f27a8b456b2a9ed --hash=sha256:454d9b4ae7881afbc25015b8627c16d88a597479b9dea82b8c6e7e2e07240dc7 --hash=sha256:4aecb6f461316adf9f1f0f6a4a1a3d79e045f9b71ec76055a791affa3b285850 --hash=sha256:4bf146dca15cdd53224a1bf46d628bd7590e4a07fbb69e720d561aea43a32b38 --hash=sha256:4c5238d32f3c5269d9e87be0cf096437b7622b6920f5eac4fd202468aaeb34d2 --hash=sha256:4e1e592789704459900728d88d41a46fe3969b82ab62945560a31732ffc19a6d --hash=sha256:509dc827f89c15c66a0c216331260d777dd6c81e9a4e4f830e662b0bb296c313 --hash=sha256:51c1c1847128238f54930edb8805b660305dca164645a9fd29243f5610beea34 --hash=sha256:5cf77eac15bd264986c4a2c63353212c095b40f3affb2bc6b4ef80c4776c1a28 --hash=sha256:5d9903ca42bfeec4cebedba8022a7c97ad2aab22e09573ce9976ba01b65e4361 --hash=sha256:61a08bcb0ec14ff4e0ed2044aad948d0659604f824cbd50b55e30b0ec6f09c73 --hash=sha256:62ba394a3dda9ad41c7c780f60f6e4a70988741415ae96f6d1bf6c239cf01379 --hash=sha256:639431bdc89d6429f6721625e8129413980ccd62e9d3f496be618a41d205f160 --hash=sha256:64350685ff08b1d3a6fff33f45a9ca183dc1d58bbfe4981604e70ec9801bbc26 --hash=sha256:6538241f45eb5a25aa575dbba1069ad786f68a4f2773a29a2bd3dd1f9de787be --hash=sha256:669dcfb2e38f9e8c69507bace46f4889e3abbfd9b0c29719202883c0a603598f --hash=sha256:66d559b21d3640203ab9075797a55165d79017520685fb407b9234d72ab63c62 --hash=sha256:6dd329a1b61c0ee95ba95385fb0c07ea0d3fe1a21e1349fa2bec272636217118 --hash=sha256:728a9d2d173a65b62bdc380b7932dd8e74ed4295279a8fe1021204ce210803e7 --hash=sha256:732aea6de26051af97b94bc98ed86448821f839d058e5d259c72bf6d73ad0fc0 --hash=sha256:74d04244852ff73b32eeede4f76f51c5bcf44bc3c207bc3e6cf1c5c45b890708 --hash=sha256:7521684c8c7c4f6e88e35ec89680ee1aa8358d3f09d27dfbdf62c446f5d4c695 --hash=sha256:75fa6f0056e7efb1f42a1c34e58be24072cb9e61a601340cc1196ae92326a4f9 --hash=sha256:78c2d02bb6e1da0720eedc0bad578049cad3f71050ef8cd065ecc87691bed2b0 --hash=sha256:795ea137b1d809eb6836b43748b12634291c0ed55ad50a7d72d21edf1cd565c4 --hash=sha256:7a50cd39f73faa34ec18d6720ee25ef10c4c1839514186fcda658a06c06057a2 --hash=sha256:7a7c7fdf755032ffdd72c77e3d8096bdcb0eb92e89e17571a196f03d88b11b3c --hash=sha256:7be0277469bf3bd7a34a9c57c1b6a724532a0d235cd0dc4e7f4316f982c28b19 --hash=sha256:7eb542fd347ce61e1321b0a6b945d5701528dca0cd9759c2e3bb8bd57e47964d --hash=sha256:7fe6e5440584e94cc4b3f5f4d98a25e29ca12dccf8873679a635638349831b98 --hash=sha256:81519e25707fc076978c6143b81ea3dc853f176895af05bf7ec51effe818aeec --hash=sha256:838441333bc90b829406d4a03cb4b8bf7656231b84358628b0406d803931ef32 --hash=sha256:849202cd789e5f3cf5dcc7822c34b502181b4824a65ff20ce82da5524e45e8e9 --hash=sha256:856a25c73b697f2ce2a24e7968285579e62577a048526161a2c0f53090bea9f9 --hash=sha256:87eb52a81ef58c7ba4d45c3ca74e12aa4b4e77816f72ca25258a85b3ea96cb48 --hash=sha256:885b26aa3ee56433b630502dc3d36ba78d186a00cc535d3806e6bfd9ed3c70ab --hash=sha256:8a3d571bd95fade53c86c0517f859477ff3a93c3fde10c9e669086f038e0f207 --hash=sha256:8e026094aa12b43f4fd74576714e987803a315c76edb6b098b9809db5de58f74 --hash=sha256:9697a52e57576c83139d7c6f213d64485d3df5bf84807c35fa409e6c970801c6 --hash=sha256:9b5aca4d5dfd7fbfbfbdaf44850fcc7709a01146a797536a8f84952e940cca76 --hash=sha256:9ddc42e68114e161e51e272f667d640f97e84a2b9ef14b7477c53aac20c2d59a --hash=sha256:9f95fbaa0ee1610ec0fc6b26668e9917a582ba80c52cc6d9ada15e30aa9ab9ad --hash=sha256:a12ab1f5c29b4e93db518f5e3872116b7e9b1646c9f9f426f777b50d44a09e8c --hash=sha256:a295ca2bba5c1c885826ce3125fa0b9f702a1be547d821c01d65f199e10c01e2 --hash=sha256:a4cb042b615245d5ff9b3794f56be4138b5adc35a4166014d31d1814744148c7 --hash=sha256:adad1a1bcf1c9e76346e091d22d23ac54ef28e1365117d99521631078dfec9de --hash=sha256:b4774ff32f18e0504bfc4e59a3e71e18d83bc1e171a3c8ed75013958a03b2f14 --hash=sha256:b6f78f98741dcc89607c16b1e9426ee46ce4bf31ac5e6b0d40e81c89f3481ea5 --hash=sha256:b7f9ee819f94c6abfa56ec7b1dbab586f41ebbdc0a57e6524bd5e7f487a878c7 --hash=sha256:ba0d8a5d7f04f73ee7d01d974d47c5834f8a1b0224390e4fe7c12a3a92a78ecc --hash=sha256:bac4200befe50c670c405dc33af26dad5a3b6b255dd6c000d92fe4629f9ed6a5 --hash=sha256:bc8ab71e2e31b16e40868a40a69007bc305e1109bd4658eb6cad007e0bf67c41 --hash=sha256:bce22519c989bb72a7e6b36a199384c53db7722fe669ba891da75907fe3587db --hash=sha256:bf3490bcbb985a1ae97b2ce9ad1c0f06a852d5b19dde9b07bdf25bf224248c95 --hash=sha256:c1e448051717a334891f2b9a620fe36776ebf3dd8ec46a0b877c8ae69575feb4 --hash=sha256:c54f768482cef41e219720013cd05933b6f971d9562544d691c68699bf2b6801 --hash=sha256:c56b4d162ca2b43318ac671c65bd4d563e841a694ac70e1a976ac38fcf4ca1d2 --hash=sha256:c9c30003b9347c24bcc210958c5d167b9e4f9be786cb380a7d32f14f9b84674f --hash=sha256:cc4076a5b4f36d849fd709284b4a3b112326652f3b0466f04002a6c15a0c96c1 --hash=sha256:cfe6d3f0c9e3b7e8c0c694b24d25e677776f5ca26dce46fd6b0489f9c8339391 --hash=sha256:d6c2d5919075a1f2e413c00b056ea0c2f065b3f5fe83c3d07d325ab92dce51d6 --hash=sha256:d8b4a27eebd684319bdf473d39f1d79eed36bf2cd34bd4465cdb4618d82b3d56 --hash=sha256:dbe6095001465294f13f1adcd3311e50dd84e5a71525f20a10bd16689c61ce0b --hash=sha256:dd16e78eb18ffdb25ee33a0682d17912e8cc8a770e885aeee95020046128f1ce --hash=sha256:ddd76a9f58e6a00f8772e72cff8ebcff78e022be95edf018766707c730593e1e --hash=sha256:df9eb838c44f570283712e7cff14c16329a9f0fb19ca492d21d4b7528ee6821e --hash=sha256:dfec44d532be4c07088c3de2876130ff0fbeeacaa89a137decbbb5f665855a0f --hash=sha256:e18bc3f73bd41243c9b38a6d9f2366cd0e0137a9aebe2d8ff76c5b67d4c0a3f4 --hash=sha256:e43586ce5bd28f9f285a6e729466841368c4a0353f6fd08d4ce4630843d3648a --hash=sha256:e6b49cd2aad93a1790ce9cffb18964f6d3a4b0b3dbdbd5de094b65296fce6e58 --hash=sha256:e6c7a21dffba883234baefe91bc3388e629779582038f75d2a5be918e250f0ed --hash=sha256:e721d1b46e25c481dc5ded6f4b3f66c897c58d2e8cfdf77bbced84339108b0b9 --hash=sha256:eadade04221641516fa25139273505a1c19f9bf97589a05bc4cfcd8b4a618031 --hash=sha256:ee3a83ce492074c35a74cc76cf8235d49e77b757193a5365ff86e3f2f93db9fd --hash=sha256:f117efad42068f9715677c8523ed2be1518116d1c49b1dd17987716695181efe --hash=sha256:f3b5a391c7597ffa96b41bd5cbd2ed0305f515fcbb367dfa72735679d5502364 --hash=sha256:f4ff94e58e84aedb9c9fce66d4ef9f27a190285b451420f297c9a09f2b9abee9 --hash=sha256:f99be08cfead2020c7ca6e396c13543baea32343b7a9a5780c462e323bd8872f --hash=sha256:fd0a5e563c756de210bb964789b5abe4f114dacae9104a47e1a649b910361536 --hash=sha256:feff9e54ec0dd3833d659257f5c3f5322a12eee58ffa360984b716f8b92983f4 --hash=sha256:ffcca5b9efe948ba0661e9df0fa50d2bc4b097c70b9810212d6b62f05d83b2dd +requests==2.32.5 --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf +scikit-learn==1.8.0 --hash=sha256:00d6f1d66fbcf4eba6e356e1420d33cc06c70a45bb1363cd6f6a8e4ebbbdece2 --hash=sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a --hash=sha256:146b4d36f800c013d267b29168813f7a03a43ecd2895d04861f1240b564421da --hash=sha256:15fc3b5d19cc2be65404786857f2e13c70c83dd4782676dd6814e3b89dc8f5b9 --hash=sha256:2838551e011a64e3053ad7618dda9310175f7515f1742fa2d756f7c874c05961 --hash=sha256:29ffc74089f3d5e87dfca4c2c8450f88bdc61b0fc6ed5d267f3988f19a1309f6 --hash=sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271 --hash=sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809 --hash=sha256:3bad7565bc9cf37ce19a7c0d107742b320c1285df7aab1a6e2d28780df167242 --hash=sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4 --hash=sha256:4511be56637e46c25721e83d1a9cea9614e7badc7040c4d573d75fbe257d6fd7 --hash=sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76 --hash=sha256:56079a99c20d230e873ea40753102102734c5953366972a71d5cb39a32bc40c6 --hash=sha256:5e30adb87f0cc81c7690a84f7932dd66be5bac57cfe16b91cb9151683a4a2d3b --hash=sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e --hash=sha256:6b595b07a03069a2b1740dc08c2299993850ea81cce4fe19b2421e0c970de6b7 --hash=sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e --hash=sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57 --hash=sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735 --hash=sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb --hash=sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb --hash=sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e --hash=sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd --hash=sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a --hash=sha256:a69525355a641bf8ef136a7fa447672fb54fe8d60cab5538d9eb7c6438543fb9 --hash=sha256:ada8121bcb4dac28d930febc791a69f7cb1673c8495e5eee274190b73a4559c1 --hash=sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde --hash=sha256:c22a2da7a198c28dd1a6e1136f19c830beab7fdca5b3e5c8bba8394f8a5c45b3 --hash=sha256:c2656924ec73e5939c76ac4c8b026fc203b83d8900362eb2599d8aee80e4880f --hash=sha256:c57b1b610bd1f40ba43970e11ce62821c2e6569e4d74023db19c6b26f246cb3b --hash=sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3 --hash=sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e --hash=sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702 --hash=sha256:f28dd15c6bb0b66ba09728cf09fd8736c304be29409bd8445a080c1280619e8c --hash=sha256:f984ca4b14914e6b4094c5d52a32ea16b49832c03bd17a110f004db3c223e8e1 --hash=sha256:fb65db5d7531bccf3a4f6bec3462223bea71384e2cda41da0f10b7c292b9e7c4 --hash=sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd +scipy==1.16.3 --hash=sha256:0151a0749efeaaab78711c78422d413c583b8cdd2011a3c1d6c794938ee9fdb2 --hash=sha256:01e87659402762f43bd2fee13370553a17ada367d42e7487800bf2916535aecb --hash=sha256:03192a35e661470197556de24e7cb1330d84b35b94ead65c46ad6f16f6b28f2a --hash=sha256:0553371015692a898e1aa858fed67a3576c34edefa6b7ebdb4e9dde49ce5c203 --hash=sha256:062246acacbe9f8210de8e751b16fc37458213f124bef161a5a02c7a39284304 --hash=sha256:0c3b4dd3d9b08dbce0f3440032c52e9e2ab9f96ade2d3943313dfe51a7056959 --hash=sha256:0c623a54f7b79dd88ef56da19bc2873afec9673a48f3b85b18e4d402bdd29a5a --hash=sha256:16b8bc35a4cc24db80a0ec836a9286d0e31b2503cb2fd7ff7fb0e0374a97081d --hash=sha256:1fb2472e72e24d1530debe6ae078db70fb1605350c88a3d14bc401d6306dbffe --hash=sha256:21d9d6b197227a12dcbf9633320a4e34c6b0e51c57268df255a0942983bac562 --hash=sha256:2a207a6ce9c24f1951241f4693ede2d393f59c07abc159b2cb2be980820e01fb --hash=sha256:2b71d93c8a9936046866acebc915e2af2e292b883ed6e2cbe5c34beb094b82d9 --hash=sha256:2d1ae2cf0c350e7705168ff2429962a89ad90c2d49d1dd300686d8b2a5af22fc --hash=sha256:3a4c460301fb2cffb7f88528f30b3127742cff583603aa7dc964a52c463b385d --hash=sha256:3d4a07a8e785d80289dfe66b7c27d8634a773020742ec7187b85ccc4b0e7b686 --hash=sha256:40be6cf99e68b6c4321e9f8782e7d5ff8265af28ef2cd56e9c9b2638fa08ad97 --hash=sha256:4aff59800a3b7f786b70bfd6ab551001cb553244988d7d6b8299cb1ea653b353 --hash=sha256:50a3dbf286dbc7d84f176f9a1574c705f277cb6565069f88f60db9eafdbe3ee2 --hash=sha256:532fb5ad6a87e9e9cd9c959b106b73145a03f04c7d57ea3e6f6bb60b86ab0876 --hash=sha256:53c3844d527213631e886621df5695d35e4f6a75f620dca412bcd292f6b87d78 --hash=sha256:56edc65510d1331dae01ef9b658d428e33ed48b4f77b1d51caf479a0253f96dc --hash=sha256:57d01cb6f85e34f0946b33caa66e892aae072b64b034183f3d87c4025802a119 --hash=sha256:5803c5fadd29de0cf27fa08ccbfe7a9e5d741bf63e4ab1085437266f12460ff9 --hash=sha256:6020470b9d00245926f2d5bb93b119ca0340f0d564eb6fbaad843eaebf9d690f --hash=sha256:63d3cdacb8a824a295191a723ee5e4ea7768ca5ca5f2838532d9f2e2b3ce2135 --hash=sha256:663b8d66a8748051c3ee9c96465fb417509315b99c71550fda2591d7dd634234 --hash=sha256:72d1717fd3b5e6ec747327ce9bda32d5463f472c9dce9f54499e81fbd50245a1 --hash=sha256:7dc1360c06535ea6116a2220f760ae572db9f661aba2d88074fe30ec2aa1ff88 --hash=sha256:7f68154688c515cdb541a31ef8eb66d8cd1050605be9dcd74199cbd22ac739bc --hash=sha256:81fc5827606858cf71446a5e98715ba0e11f0dbc83d71c7409d05486592a45d6 --hash=sha256:875555ce62743e1d54f06cdf22c1e0bc47b91130ac40fe5d783b6dfa114beeb6 --hash=sha256:8b3c820ddb80029fe9f43d61b81d8b488d3ef8ca010d15122b152db77dc94c22 --hash=sha256:8be1ca9170fcb6223cc7c27f4305d680ded114a1567c0bd2bfcbf947d1b17511 --hash=sha256:8d09d72dc92742988b0e7750bddb8060b0c7079606c0d24a8cc8e9c9c11f9079 --hash=sha256:9452781bd879b14b6f055b26643703551320aa8d79ae064a71df55c00286a184 --hash=sha256:96491a6a54e995f00a28a3c3badfff58fd093bf26cd5fb34a2188c8c756a3a2c --hash=sha256:9b9c9c07b6d56a35777a1b4cc8966118fb16cfd8daf6743867d17d36cfad2d40 --hash=sha256:a8a26c78ef223d3e30920ef759e25625a0ecdd0d60e5a8818b7513c3e5384cf2 --hash=sha256:aadd23f98f9cb069b3bd64ddc900c4d277778242e961751f77a8cb5c4b946fb0 --hash=sha256:b7180967113560cca57418a7bc719e30366b47959dd845a93206fbed693c867e --hash=sha256:b7c5f1bda1354d6a19bc6af73a649f8285ca63ac6b52e64e658a5a11d4d69800 --hash=sha256:b81c27fc41954319a943d43b20e07c40bdcd3ff7cf013f4fb86286faefe546c4 --hash=sha256:bb61878c18a470021fb515a843dc7a76961a8daceaaaa8bad1332f1bf4b54657 --hash=sha256:bea0a62734d20d67608660f69dcda23e7f90fb4ca20974ab80b6ed40df87a005 --hash=sha256:c5192722cffe15f9329a3948c4b1db789fbb1f05c97899187dcf009b283aea70 --hash=sha256:c97176013d404c7346bf57874eaac5187d969293bf40497140b0a2b2b7482e07 --hash=sha256:cd13e354df9938598af2be05822c323e97132d5e6306b83a3b4ee6724c6e522e --hash=sha256:d2ec56337675e61b312179a1ad124f5f570c00f920cc75e1000025451b88241c --hash=sha256:d3837938ae715fc0fe3c39c0202de3a8853aff22ca66781ddc2ade7554b7e2cc --hash=sha256:d9f48cafc7ce94cf9b15c6bffdc443a81a27bf7075cf2dcd5c8b40f85d10c4e7 --hash=sha256:da7763f55885045036fabcebd80144b757d3db06ab0861415d1c3b7c69042146 --hash=sha256:deb3841c925eeddb6afc1e4e4a45e418d19ec7b87c5df177695224078e8ec733 --hash=sha256:e1d27cbcb4602680a49d787d90664fa4974063ac9d4134813332a8c53dbe667c --hash=sha256:e5d42a9472e7579e473879a1990327830493a7047506d58d73fc429b84c1d49d --hash=sha256:e7efa2681ea410b10dde31a52b18b0154d66f2485328830e45fdf183af5aefc6 --hash=sha256:eab43fae33a0c39006a88096cd7b4f4ef545ea0447d250d5ac18202d40b6611d --hash=sha256:f2622206f5559784fa5c4b53a950c3c7c1cf3e84ca1b9c4b6c03f062f289ca26 --hash=sha256:f379b54b77a597aa7ee5e697df0d66903e41b9c85a6dd7946159e356319158e8 --hash=sha256:f667a4542cc8917af1db06366d3f78a5c8e83badd56409f94d1eac8d8d9133fa --hash=sha256:fb4b29f4cf8cc5a8d628bc8d8e26d12d7278cd1f219f22698a378c3d67db5e4b --hash=sha256:ffa6eea95283b2b8079b821dc11f50a17d0571c92b43e2b5b12764dc5f9b285d +setuptools==80.9.0 --hash=sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922 --hash=sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c +six==1.17.0 --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 +sympy==1.14.0 --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 +threadpoolctl==3.6.0 --hash=sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb --hash=sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e +tiktoken==0.12.0 --hash=sha256:01d99484dc93b129cd0964f9d34eee953f2737301f18b3c7257bf368d7615baa --hash=sha256:04f0e6a985d95913cabc96a741c5ffec525a2c72e9df086ff17ebe35985c800e --hash=sha256:06a9f4f49884139013b138920a4c393aa6556b2f8f536345f11819389c703ebb --hash=sha256:09eb4eae62ae7e4c62364d9ec3a57c62eea707ac9a2b2c5d6bd05de6724ea179 --hash=sha256:0ee8f9ae00c41770b5f9b0bb1235474768884ae157de3beb5439ca0fd70f3e25 --hash=sha256:15d875454bbaa3728be39880ddd11a5a2a9e548c29418b41e8fd8a767172b5ec --hash=sha256:20cf97135c9a50de0b157879c3c4accbb29116bcf001283d26e073ff3b345946 --hash=sha256:285ba9d73ea0d6171e7f9407039a290ca77efcdb026be7769dccc01d2c8d7fff --hash=sha256:2b90f5ad190a4bb7c3eb30c5fa32e1e182ca1ca79f05e49b448438c3e225a49b --hash=sha256:2cff3688ba3c639ebe816f8d58ffbbb0aa7433e23e08ab1cade5d175fc973fb3 --hash=sha256:35a2f8ddd3824608b3d650a000c1ef71f730d0c56486845705a8248da00f9fe5 --hash=sha256:399c3dd672a6406719d84442299a490420b458c44d3ae65516302a99675888f3 --hash=sha256:3de02f5a491cfd179aec916eddb70331814bd6bf764075d39e21d5862e533970 --hash=sha256:3e68e3e593637b53e56f7237be560f7a394451cb8c11079755e80ae64b9e6def --hash=sha256:47a5bc270b8c3db00bb46ece01ef34ad050e364b51d406b6f9730b64ac28eded --hash=sha256:4a1a4fcd021f022bfc81904a911d3df0f6543b9e7627b51411da75ff2fe7a1be --hash=sha256:4c9614597ac94bb294544345ad8cf30dac2129c05e2db8dc53e082f355857af7 --hash=sha256:508fa71810c0efdcd1b898fda574889ee62852989f7c1667414736bcb2b9a4bd --hash=sha256:54c891b416a0e36b8e2045b12b33dd66fb34a4fe7965565f1b482da50da3e86a --hash=sha256:584c3ad3d0c74f5269906eb8a659c8bfc6144a52895d9261cdaf90a0ae5f4de0 --hash=sha256:5edb8743b88d5be814b1a8a8854494719080c28faaa1ccbef02e87354fe71ef0 --hash=sha256:604831189bd05480f2b885ecd2d1986dc7686f609de48208ebbbddeea071fc0b --hash=sha256:65b26c7a780e2139e73acc193e5c63ac754021f160df919add909c1492c0fb37 --hash=sha256:6de0da39f605992649b9cfa6f84071e3f9ef2cec458d08c5feb1b6f0ff62e134 --hash=sha256:6e227c7f96925003487c33b1b32265fad2fbcec2b7cf4817afb76d416f40f6bb --hash=sha256:6faa0534e0eefbcafaccb75927a4a380463a2eaa7e26000f0173b920e98b720a --hash=sha256:6fb2995b487c2e31acf0a9e17647e3b242235a20832642bb7a9d1a181c0c1bb1 --hash=sha256:775c2c55de2310cc1bc9a3ad8826761cbdc87770e586fd7b6da7d4589e13dab3 --hash=sha256:82991e04fc860afb933efb63957affc7ad54f83e2216fe7d319007dab1ba5892 --hash=sha256:83d16643edb7fa2c99eff2ab7733508aae1eebb03d5dfc46f5565862810f24e3 --hash=sha256:8f317e8530bb3a222547b85a58583238c8f74fd7a7408305f9f63246d1a0958b --hash=sha256:981a81e39812d57031efdc9ec59fa32b2a5a5524d20d4776574c4b4bd2e9014a --hash=sha256:9baf52f84a3f42eef3ff4e754a0db79a13a27921b457ca9832cf944c6be4f8f3 --hash=sha256:a01b12f69052fbe4b080a2cfb867c4de12c704b56178edf1d1d7b273561db160 --hash=sha256:a1af81a6c44f008cba48494089dd98cccb8b313f55e961a52f5b222d1e507967 --hash=sha256:a90388128df3b3abeb2bfd1895b0681412a8d7dc644142519e6f0a97c2111646 --hash=sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931 --hash=sha256:b4e7ed1c6a7a8a60a3230965bdedba8cc58f68926b835e519341413370e0399a --hash=sha256:b6cfb6d9b7b54d20af21a912bfe63a2727d9cfa8fbda642fd8322c70340aad16 --hash=sha256:b8a0cd0c789a61f31bf44851defbd609e8dd1e2c8589c614cc1060940ef1f697 --hash=sha256:b97f74aca0d78a1ff21b8cd9e9925714c15a9236d6ceacf5c7327c117e6e21e8 --hash=sha256:c06cf0fcc24c2cb2adb5e185c7082a82cba29c17575e828518c2f11a01f445aa --hash=sha256:c2c714c72bc00a38ca969dae79e8266ddec999c7ceccd603cc4f0d04ccd76365 --hash=sha256:cbb9a3ba275165a2cb0f9a83f5d7025afe6b9d0ab01a22b50f0e74fee2ad253e --hash=sha256:cde24cdb1b8a08368f709124f15b36ab5524aac5fa830cc3fdce9c03d4fb8030 --hash=sha256:d186a5c60c6a0213f04a7a802264083dea1bbde92a2d4c7069e1a56630aef830 --hash=sha256:d51d75a5bffbf26f86554d28e78bfb921eae998edc2675650fd04c7e1f0cdc1e --hash=sha256:d5f89ea5680066b68bcb797ae85219c72916c922ef0fcdd3480c7d2315ffff16 --hash=sha256:da900aa0ad52247d8794e307d6446bd3cdea8e192769b56276695d34d2c9aa88 --hash=sha256:dc2dd125a62cb2b3d858484d6c614d136b5b848976794edfb63688d539b8b93f --hash=sha256:df37684ace87d10895acb44b7f447d4700349b12197a526da0d4a4149fde074c --hash=sha256:dfdfaa5ffff8993a3af94d1125870b1d27aed7cb97aa7eb8c1cefdbc87dbee63 --hash=sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad --hash=sha256:f18f249b041851954217e9fd8e5c00b024ab2315ffda5ed77665a05fa91f42dc --hash=sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71 --hash=sha256:fc530a28591a2d74bce821d10b418b26a094bf33839e69042a6e86ddb7a7fb27 --hash=sha256:ffc5288f34a8bc02e1ea7047b8d041104791d2ddbf42d1e5fa07822cbffe16bd +torch @ https://download-r2.pytorch.org/whl/cu128/torch-2.9.1%2Bcu128-cp311-cp311-manylinux_2_28_x86_64.whl --hash=sha256:2a1da940f0757621d098c9755f7504d791a72a40920ec85a4fd98b20253fca4e +torchmetrics==1.8.2 --hash=sha256:08382fd96b923e39e904c4d570f3d49e2cc71ccabd2a94e0f895d1f0dac86242 --hash=sha256:cf64a901036bf107f17a524009eea7781c9c5315d130713aeca5747a686fe7a5 +tqdm==4.67.1 --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 +triton==3.5.1 --hash=sha256:02c770856f5e407d24d28ddc66e33cf026e6f4d360dcb8b2fabe6ea1fc758621 --hash=sha256:0b4d2c70127fca6a23e247f9348b8adde979d2e7a20391bfbabaac6aebc7e6a8 --hash=sha256:275a045b6ed670dd1bd005c3e6c2d61846c74c66f4512d6f33cc027b11de8fd4 --hash=sha256:56765ffe12c554cd560698398b8a268db1f616c120007bfd8829d27139abd24a --hash=sha256:5fc53d849f879911ea13f4a877243afc513187bc7ee92d1f2c0f1ba3169e3c94 --hash=sha256:61413522a48add32302353fdbaaf92daaaab06f6b5e3229940d21b5207f47579 --hash=sha256:8932391d7f93698dfe5bc9bead77c47a24f97329e9f20c10786bb230a9083f56 --hash=sha256:bac7f7d959ad0f48c0e97d6643a1cc0fd5786fe61cb1f83b537c6b2d54776478 --hash=sha256:d0637b1efb1db599a8e9dc960d53ab6e4637db7d4ab6630a0974705d77b14b60 --hash=sha256:d2c6b915a03888ab931a9fd3e55ba36785e1fe70cbea0b40c6ef93b20fc85232 --hash=sha256:da47169e30a779bade679ce78df4810fca6d78a955843d2ddb11f226adc517dc --hash=sha256:f3f4346b6ebbd4fad18773f5ba839114f4826037c9f2f34e0148894cd5dd3dba --hash=sha256:f617aa7925f9ea9968ec2e1adaf93e87864ff51549c8f04ce658f29bbdb71e2d --hash=sha256:f63e34dcb32d7bd3a1d0195f60f30d2aee8b08a69a0424189b71017e23dfc3d2 +typing-extensions==4.15.0 --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 +tzdata==2025.3 --hash=sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1 --hash=sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7 +urllib3==2.6.3 --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 +yarl==1.22.0 --hash=sha256:01e73b85a5434f89fc4fe27dcda2aff08ddf35e4d47bbbea3bdcd25321af538a --hash=sha256:029866bde8d7b0878b9c160e72305bbf0a7342bcd20b9999381704ae03308dc8 --hash=sha256:078278b9b0b11568937d9509b589ee83ef98ed6d561dfe2020e24a9fd08eaa2b --hash=sha256:078a8aefd263f4d4f923a9677b942b445a2be970ca24548a8102689a3a8ab8da --hash=sha256:07a524d84df0c10f41e3ee918846e1974aba4ec017f990dc735aad487a0bdfdf --hash=sha256:088e4e08f033db4be2ccd1f34cf29fe994772fb54cfe004bbf54db320af56890 --hash=sha256:0b5bcc1a9c4839e7e30b7b30dd47fe5e7e44fb7054ec29b5bb8d526aa1041093 --hash=sha256:0cf71bf877efeac18b38d3930594c0948c82b64547c1cf420ba48722fe5509f6 --hash=sha256:0d6e6885777af0f110b0e5d7e5dda8b704efed3894da26220b7f3d887b839a79 --hash=sha256:0dd9a702591ca2e543631c2a017e4a547e38a5c0f29eece37d9097e04a7ac683 --hash=sha256:10619d9fdee46d20edc49d3479e2f8269d0779f1b031e6f7c2aa1c76be04b7ed --hash=sha256:131a085a53bfe839a477c0845acf21efc77457ba2bcf5899618136d64f3303a2 --hash=sha256:1380560bdba02b6b6c90de54133c81c9f2a453dee9912fe58c1dcced1edb7cff --hash=sha256:139718f35149ff544caba20fce6e8a2f71f1e39b92c700d8438a0b1d2a631a02 --hash=sha256:14291620375b1060613f4aab9ebf21850058b6b1b438f386cc814813d901c60b --hash=sha256:1834bb90991cc2999f10f97f5f01317f99b143284766d197e43cd5b45eb18d03 --hash=sha256:1ab72135b1f2db3fed3997d7e7dc1b80573c67138023852b6efb336a5eae6511 --hash=sha256:1e7ce67c34138a058fd092f67d07a72b8e31ff0c9236e751957465a24b28910c --hash=sha256:1e8fbaa7cec507aa24ea27a01456e8dd4b6fab829059b69844bd348f2d467124 --hash=sha256:22965c2af250d20c873cdbee8ff958fb809940aeb2e74ba5f20aaf6b7ac8c70c --hash=sha256:22b029f2881599e2f1b06f8f1db2ee63bd309e2293ba2d566e008ba12778b8da --hash=sha256:243dda95d901c733f5b59214d28b0120893d91777cb8aa043e6ef059d3cddfe2 --hash=sha256:2ca6fd72a8cd803be290d42f2dec5cdcd5299eeb93c2d929bf060ad9efaf5de0 --hash=sha256:2e4e1f6f0b4da23e61188676e3ed027ef0baa833a2e633c29ff8530800edccba --hash=sha256:31f0b53913220599446872d757257be5898019c85e7971599065bc55065dc99d --hash=sha256:334b8721303e61b00019474cc103bdac3d7b1f65e91f0bfedeec2d56dfe74b53 --hash=sha256:33e32a0dd0c8205efa8e83d04fc9f19313772b78522d1bdc7d9aed706bfd6138 --hash=sha256:34b36c2c57124530884d89d50ed2c1478697ad7473efd59cfd479945c95650e4 --hash=sha256:3aa27acb6de7a23785d81557577491f6c38a5209a254d1191519d07d8fe51748 --hash=sha256:3b06bcadaac49c70f4c88af4ffcfbe3dc155aab3163e75777818092478bcbbe7 --hash=sha256:3b7c88eeef021579d600e50363e0b6ee4f7f6f728cd3486b9d0f3ee7b946398d --hash=sha256:3e2daa88dc91870215961e96a039ec73e4937da13cf77ce17f9cad0c18df3503 --hash=sha256:3ea66b1c11c9150f1372f69afb6b8116f2dd7286f38e14ea71a44eee9ec51b9d --hash=sha256:42188e6a615c1a75bcaa6e150c3fe8f3e8680471a6b10150c5f7e83f47cc34d2 --hash=sha256:433885ab5431bc3d3d4f2f9bd15bfa1614c522b0f1405d62c4f926ccd69d04fa --hash=sha256:437840083abe022c978470b942ff832c3940b2ad3734d424b7eaffcd07f76737 --hash=sha256:4398557cbf484207df000309235979c79c4356518fd5c99158c7d38203c4da4f --hash=sha256:45c2842ff0e0d1b35a6bf1cd6c690939dacb617a70827f715232b2e0494d55d1 --hash=sha256:47743b82b76d89a1d20b83e60d5c20314cbd5ba2befc9cda8f28300c4a08ed4d --hash=sha256:4792b262d585ff0dff6bcb787f8492e40698443ec982a3568c2096433660c694 --hash=sha256:47d8a5c446df1c4db9d21b49619ffdba90e77c89ec6e283f453856c74b50b9e3 --hash=sha256:47fdb18187e2a4e18fda2c25c05d8251a9e4a521edaed757fef033e7d8498d9a --hash=sha256:4c52a6e78aef5cf47a98ef8e934755abf53953379b7d53e68b15ff4420e6683d --hash=sha256:4dcc74149ccc8bba31ce1944acee24813e93cfdee2acda3c172df844948ddf7b --hash=sha256:50678a3b71c751d58d7908edc96d332af328839eea883bb554a43f539101277a --hash=sha256:51af598701f5299012b8416486b40fceef8c26fc87dc6d7d1f6fc30609ea0aa6 --hash=sha256:594fcab1032e2d2cc3321bb2e51271e7cd2b516c7d9aee780ece81b07ff8244b --hash=sha256:595697f68bd1f0c1c159fcb97b661fc9c3f5db46498043555d04805430e79bea --hash=sha256:59c189e3e99a59cf8d83cbb31d4db02d66cda5a1a4374e8a012b51255341abf5 --hash=sha256:5a3bf7f62a289fa90f1990422dc8dff5a458469ea71d1624585ec3a4c8d6960f --hash=sha256:5c401e05ad47a75869c3ab3e35137f8468b846770587e70d71e11de797d113df --hash=sha256:5cdac20da754f3a723cceea5b3448e1a2074866406adeb4ef35b469d089adb8f --hash=sha256:5d0fcda9608875f7d052eff120c7a5da474a6796fe4d83e152e0e4d42f6d1a9b --hash=sha256:5dbeefd6ca588b33576a01b0ad58aa934bc1b41ef89dee505bf2932b22ddffba --hash=sha256:62441e55958977b8167b2709c164c91a6363e25da322d87ae6dd9c6019ceecf9 --hash=sha256:663e1cadaddae26be034a6ab6072449a8426ddb03d500f43daf952b74553bba0 --hash=sha256:669930400e375570189492dc8d8341301578e8493aec04aebc20d4717f899dd6 --hash=sha256:68986a61557d37bb90d3051a45b91fa3d5c516d177dfc6dd6f2f436a07ff2b6b --hash=sha256:6944b2dc72c4d7f7052683487e3677456050ff77fcf5e6204e98caf785ad1967 --hash=sha256:6a635ea45ba4ea8238463b4f7d0e721bad669f80878b7bfd1f89266e2ae63da2 --hash=sha256:6c5010a52015e7c70f86eb967db0f37f3c8bd503a695a49f8d45700144667708 --hash=sha256:6dcbb0829c671f305be48a7227918cfcd11276c2d637a8033a99a02b67bf9eda --hash=sha256:70dfd4f241c04bd9239d53b17f11e6ab672b9f1420364af63e8531198e3f5fe8 --hash=sha256:719ae08b6972befcba4310e49edb1161a88cdd331e3a694b84466bd938a6ab10 --hash=sha256:75976c6945d85dbb9ee6308cd7ff7b1fb9409380c82d6119bd778d8fcfe2931c --hash=sha256:7861058d0582b847bc4e3a4a4c46828a410bca738673f35a29ba3ca5db0b473b --hash=sha256:792a2af6d58177ef7c19cbf0097aba92ca1b9cb3ffdd9c7470e156c8f9b5e028 --hash=sha256:8009b3173bcd637be650922ac455946197d858b3630b6d8787aa9e5c4564533e --hash=sha256:80ddf7a5f8c86cb3eb4bc9028b07bbbf1f08a96c5c0bc1244be5e8fefcb94147 --hash=sha256:8218f4e98d3c10d683584cb40f0424f4b9fd6e95610232dd75e13743b070ee33 --hash=sha256:84fc3ec96fce86ce5aa305eb4aa9358279d1aa644b71fab7b8ed33fe3ba1a7ca --hash=sha256:852863707010316c973162e703bddabec35e8757e67fcb8ad58829de1ebc8590 --hash=sha256:8884d8b332a5e9b88e23f60bb166890009429391864c685e17bd73a9eda9105c --hash=sha256:8dee9c25c74997f6a750cd317b8ca63545169c098faee42c84aa5e506c819b53 --hash=sha256:939fe60db294c786f6b7c2d2e121576628468f65453d86b0fe36cb52f987bd74 --hash=sha256:99b6fc1d55782461b78221e95fc357b47ad98b041e8e20f47c1411d0aacddc60 --hash=sha256:9d7672ecf7557476642c88497c2f8d8542f8e36596e928e9bcba0e42e1e7d71f --hash=sha256:9f6d73c1436b934e3f01df1e1b21ff765cd1d28c77dfb9ace207f746d4610ee1 --hash=sha256:9fb17ea16e972c63d25d4a97f016d235c78dd2344820eb35bc034bc32012ee27 --hash=sha256:a49370e8f711daec68d09b821a34e1167792ee2d24d405cbc2387be4f158b520 --hash=sha256:a4fcfc8eb2c34148c118dfa02e6427ca278bfd0f3df7c5f99e33d2c0e81eae3e --hash=sha256:a899cbd98dce6f5d8de1aad31cb712ec0a530abc0a86bd6edaa47c1090138467 --hash=sha256:a9b1ba5610a4e20f655258d5a1fdc7ebe3d837bb0e45b581398b99eb98b1f5ca --hash=sha256:af74f05666a5e531289cb1cc9c883d1de2088b8e5b4de48004e5ca8a830ac859 --hash=sha256:b0748275abb8c1e1e09301ee3cf90c8a99678a4e92e4373705f2a2570d581273 --hash=sha256:b266bd01fedeffeeac01a79ae181719ff848a5a13ce10075adbefc8f1daee70e --hash=sha256:b4f15793aa49793ec8d1c708ab7f9eded1aa72edc5174cae703651555ed1b601 --hash=sha256:b580e71cac3f8113d3135888770903eaf2f507e9421e5697d6ee6d8cd1c7f054 --hash=sha256:b6a6f620cfe13ccec221fa312139135166e47ae169f8253f72a0abc0dae94376 --hash=sha256:b790b39c7e9a4192dc2e201a282109ed2985a1ddbd5ac08dc56d0e121400a8f7 --hash=sha256:b85b982afde6df99ecc996990d4ad7ccbdbb70e2a4ba4de0aecde5922ba98a0b --hash=sha256:b8a0588521a26bf92a57a1705b77b8b59044cdceccac7151bd8d229e66b8dedb --hash=sha256:ba440ae430c00eee41509353628600212112cd5018d5def7e9b05ea7ac34eb65 --hash=sha256:bca03b91c323036913993ff5c738d0842fc9c60c4648e5c8d98331526df89784 --hash=sha256:bebf8557577d4401ba8bd9ff33906f1376c877aa78d1fe216ad01b4d6745af71 --hash=sha256:bec03d0d388060058f5d291a813f21c011041938a441c593374da6077fe21b1b --hash=sha256:bf4a21e58b9cde0e401e683ebd00f6ed30a06d14e93f7c8fd059f8b6e8f87b6a --hash=sha256:c0232bce2170103ec23c454e54a57008a9a72b5d1c3105dc2496750da8cfa47c --hash=sha256:c4647674b6150d2cae088fc07de2738a84b8bcedebef29802cf0b0a82ab6face --hash=sha256:c7044802eec4524fde550afc28edda0dd5784c4c45f0be151a2d3ba017daca7d --hash=sha256:c7bd6683587567e5a49ee6e336e0612bec8329be1b7d4c8af5687dcdeb67ee1e --hash=sha256:ca1f59c4e1ab6e72f0a23c13fca5430f889634166be85dbf1013683e49e3278e --hash=sha256:cb95a9b1adaa48e41815a55ae740cfda005758104049a640a398120bf02515ca --hash=sha256:cfebc0ac8333520d2d0423cbbe43ae43c8838862ddb898f5ca68565e395516e9 --hash=sha256:d332fc2e3c94dad927f2112395772a4e4fedbcf8f80efc21ed7cdfae4d574fdb --hash=sha256:d3e32536234a95f513bd374e93d717cf6b2231a791758de6c509e3653f234c95 --hash=sha256:d5372ca1df0f91a86b047d1277c2aaf1edb32d78bbcefffc81b40ffd18f027ed --hash=sha256:d77e1b2c6d04711478cb1c4ab90db07f1609ccf06a287d5607fcd90dc9863acf --hash=sha256:d947071e6ebcf2e2bee8fce76e10faca8f7a14808ca36a910263acaacef08eca --hash=sha256:dd7afd3f8b0bfb4e0d9fc3c31bfe8a4ec7debe124cfd90619305def3c8ca8cd2 --hash=sha256:de6b9a04c606978fdfe72666fa216ffcf2d1a9f6a381058d4378f8d7b1e5de62 --hash=sha256:e1651bf8e0398574646744c1885a41198eba53dc8a9312b954073f845c90a8df --hash=sha256:e1b329cb8146d7b736677a2440e422eadd775d1806a81db2d4cded80a48efc1a --hash=sha256:e1b51bebd221006d3d2f95fbe124b22b247136647ae5dcc8c7acafba66e5ee67 --hash=sha256:e340382d1afa5d32b892b3ff062436d592ec3d692aeea3bef3a5cfe11bbf8c6f --hash=sha256:e4b582bab49ac33c8deb97e058cd67c2c50dac0dd134874106d9c774fd272529 --hash=sha256:e51ac5435758ba97ad69617e13233da53908beccc6cfcd6c34bbed8dcbede486 --hash=sha256:e5542339dcf2747135c5c85f68680353d5cb9ffd741c0f2e8d832d054d41f35a --hash=sha256:e6438cc8f23a9c1478633d216b16104a586b9761db62bfacb6425bac0a36679e --hash=sha256:e81fda2fb4a07eda1a2252b216aa0df23ebcd4d584894e9612e80999a78fd95b --hash=sha256:ea70f61a47f3cc93bdf8b2f368ed359ef02a01ca6393916bc8ff877427181e74 --hash=sha256:ebd4549b108d732dba1d4ace67614b9545b21ece30937a63a65dd34efa19732d --hash=sha256:efb07073be061c8f79d03d04139a80ba33cbd390ca8f0297aae9cce6411e4c6b --hash=sha256:f0d97c18dfd9a9af4490631905a3f131a8e4c9e80a39353919e2cfed8f00aedc --hash=sha256:f1e09112a2c31ffe8d80be1b0988fa6a18c5d5cad92a9ffbb1c04c91bfe52ad2 --hash=sha256:f3d7a87a78d46a2e3d5b72587ac14b4c16952dd0887dbb051451eceac774411e --hash=sha256:f4afb5c34f2c6fecdcc182dfcfc6af6cccf1aa923eed4d6a12e9d96904e1a0d8 --hash=sha256:f6d2cb59377d99718913ad9a151030d6f83ef420a2b8f521d94609ecc106ee82 --hash=sha256:f87ac53513d22240c7d59203f25cc3beac1e574c6cd681bbfd321987b69f95fd --hash=sha256:ff86011bd159a9d2dfc89c34cfd8aff12875980e3bd6a39ff097887520e60249 +zstandard==0.25.0 --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a --hash=sha256:05353cef599a7b0b98baca9b068dd36810c3ef0f42bf282583f438caf6ddcee3 --hash=sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f --hash=sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6 --hash=sha256:07b527a69c1e1c8b5ab1ab14e2afe0675614a09182213f21a0717b62027b5936 --hash=sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431 --hash=sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250 --hash=sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa --hash=sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f --hash=sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851 --hash=sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3 --hash=sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9 --hash=sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6 --hash=sha256:19796b39075201d51d5f5f790bf849221e58b48a39a5fc74837675d8bafc7362 --hash=sha256:1cd5da4d8e8ee0e88be976c294db744773459d51bb32f707a0f166e5ad5c8649 --hash=sha256:1f3689581a72eaba9131b1d9bdbfe520ccd169999219b41000ede2fca5c1bfdb --hash=sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5 --hash=sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439 --hash=sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137 --hash=sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa --hash=sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd --hash=sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701 --hash=sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0 --hash=sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043 --hash=sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1 --hash=sha256:37daddd452c0ffb65da00620afb8e17abd4adaae6ce6310702841760c2c26860 --hash=sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611 --hash=sha256:3b870ce5a02d4b22286cf4944c628e0f0881b11b3f14667c1d62185a99e04f53 --hash=sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b --hash=sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088 --hash=sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e --hash=sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa --hash=sha256:4b14abacf83dfb5c25eb4e4a79520de9e7e205f72c9ee7702f91233ae57d33a2 --hash=sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0 --hash=sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7 --hash=sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf --hash=sha256:51526324f1b23229001eb3735bc8c94f9c578b1bd9e867a0a646a3b17109f388 --hash=sha256:53e08b2445a6bc241261fea89d065536f00a581f02535f8122eba42db9375530 --hash=sha256:53f94448fe5b10ee75d246497168e5825135d54325458c4bfffbaafabcc0a577 --hash=sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902 --hash=sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc --hash=sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98 --hash=sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a --hash=sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097 --hash=sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea --hash=sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09 --hash=sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb --hash=sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7 --hash=sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74 --hash=sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b --hash=sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b --hash=sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b --hash=sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91 --hash=sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150 --hash=sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049 --hash=sha256:89c4b48479a43f820b749df49cd7ba2dbc2b1b78560ecb5ab52985574fd40b27 --hash=sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a --hash=sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00 --hash=sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd --hash=sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072 --hash=sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c --hash=sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c --hash=sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065 --hash=sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512 --hash=sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1 --hash=sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f --hash=sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2 --hash=sha256:a51ff14f8017338e2f2e5dab738ce1ec3b5a851f23b18c1ae1359b1eecbee6df --hash=sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab --hash=sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7 --hash=sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b --hash=sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550 --hash=sha256:b9af1fe743828123e12b41dd8091eca1074d0c1569cc42e6e1eee98027f2bbd0 --hash=sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea --hash=sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277 --hash=sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2 --hash=sha256:c2ba942c94e0691467ab901fc51b6f2085ff48f2eea77b1a48240f011e8247c7 --hash=sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778 --hash=sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859 --hash=sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d --hash=sha256:d8c56bb4e6c795fc77d74d8e8b80846e1fb8292fc0b5060cd8131d522974b751 --hash=sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12 --hash=sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2 --hash=sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d --hash=sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0 --hash=sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3 --hash=sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd --hash=sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e --hash=sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f --hash=sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e --hash=sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94 --hash=sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708 --hash=sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313 --hash=sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4 --hash=sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c --hash=sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344 --hash=sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551 --hash=sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01 diff --git a/benchmarks/cqa/study.py b/benchmarks/cqa/study.py new file mode 100644 index 000000000..7c5456391 --- /dev/null +++ b/benchmarks/cqa/study.py @@ -0,0 +1,353 @@ +"""Frozen CQA inputs, reference batches, verification gates, and job execution.""" + +import importlib.metadata +import os +import platform +import random +import subprocess +import time +from collections import Counter +from pathlib import Path + +import numpy as np +import torch + +from dicee.query_answering._checkpoint import BenchmarkCheckpoint, checksum, implementation_fingerprint, input_path, read, source_fingerprint, write_json +from dicee.query_answering._query import PLUS_H_SHAPES, compile_query, relation_signature +from dicee.query_answering.context import fingerprint +from dicee.query_answering.datasets import PLUS_H_DATASETS, dataset_spec, inference_graph_for_split, load_benchmark, source_query_groups +from dicee.query_answering.method_evaluation import evaluate_method, save_results +from dicee.query_answering.methods import REFERENCES + +from .manifests import validate_entry +from .provenance import validate_training + +SOURCE_ROOT = Path(__file__).resolve().parents[2] + + +def dataset_key(entry, split): + key = f'{entry["dataset"]}/{split}' + graph = inference_graph_for_split(entry['dataset'], split, entry.get('inference_graph')) + return key + '/train+valid' if graph == 'train+valid' else key + + +def code_identity(): + """Source that can change frozen-study results. + + The dicee package without its command-line scripts, and this harness without + tests, paper tooling or the reproduction driver (whose manifests and adapters + are pinned by the bundle itself), so local scripts never break verification elsewhere. + """ + package, harness = SOURCE_ROOT / 'dicee', Path(__file__).resolve().parent + paths = [path for path in package.rglob('*.py') if path.relative_to(package).parts[0] != 'scripts'] + paths += [path for path in harness.rglob('*.py') if path.relative_to(harness).parts[0] not in ('tests', 'paper', 'reproduction')] + return source_fingerprint(paths, SOURCE_ROOT) + + +def environment(device): + try: + driver = subprocess.check_output(['nvidia-smi', '--query-gpu=uuid,name,driver_version', '--format=csv,noheader'], text=True).strip() + except (OSError, subprocess.CalledProcessError): + driver = None + return dict(python=platform.python_version(), platform=platform.platform(), torch=str(torch.__version__), + cuda=torch.version.cuda, device=str(device), driver=driver, + gpu=torch.cuda.get_device_name(device) if torch.device(device).type == 'cuda' else None, + image_id=os.environ.get('DICEE_IMAGE_ID'), source_commit=os.environ.get('DICEE_SOURCE_COMMIT'), + source_dirty=os.environ.get('DICEE_SOURCE_DIRTY'), + packages={name: importlib.metadata.version(name) for name in ('torch', 'numpy', 'lightning', 'pandas')}, + threads=torch.get_num_threads(), precision='float32 IEEE; no autocast', + deterministic_algorithms=torch.are_deterministic_algorithms_enabled(), + cublas_workspace_config=os.environ.get('CUBLAS_WORKSPACE_CONFIG')) + + +def make_plan(data, entry, root, query_ids=None): + """Keep each structure's original conversion order and batch membership.""" + wanted = set(entry['query_types']) + lookup = {query.query: query for query in data.queries if query.shape in wanted} + if entry['query_order'] == 'upstream-pickle': + groups = source_query_groups(data, input_path(root, entry['_data_root']), entry['query_types']) + else: + groups = [[query for query in data.queries if query.shape == shape] for shape in entry['query_types']] + if entry['query_order'] == 'relation': + groups = [sorted(group, key=lambda query: (relation_signature(compile_query(query.query)), query.query)) for group in groups] + queries, ends = [], [] + for group in groups: + for start in range(0, len(group), entry['query_batch_size']): + queries.extend(group[start:start + entry['query_batch_size']]) + ends.append(len(queries)) + if len(queries) != len(lookup) or len(set(queries)) != len(queries): + raise ValueError('Reference order does not cover each selected query exactly once') + return dict(queries=[query_ids[query.query] if query_ids is not None else query.identity for query in queries], batch_ends=ends, + query_counts=dict(Counter(query.shape for query in queries)), order=entry['query_order']) + + +def freeze(manifest, output, input_root): + """Pin inputs and plans without loading model weights or running test inference.""" + manifest = read(manifest) if isinstance(manifest, (str, Path)) else manifest + if (set(manifest) - {'answer_filter'} != {'version', 'data_root', 'entries', 'seed', 'threads', 'archives'} + or manifest['version'] != 1 or manifest.get('answer_filter', 'released') not in ('released', 'corrected')): + raise ValueError('Expected version, data_root, entries, seed, threads, archives') + if type(manifest['seed']) is not int or type(manifest['threads']) is not int or manifest['threads'] < 1: + raise ValueError('Invalid seed or thread count') + entries = manifest['entries'] + if not entries or len({entry['id'] for entry in entries}) != len(entries): + raise ValueError('Entries need unique IDs') + for entry in entries: + validate_entry(entry) + if manifest.get('answer_filter') == 'corrected' and any(e['dataset'] not in PLUS_H_DATASETS for e in entries): + raise ValueError('Corrected answer filters are only defined for the +H suite') + output = Path(output) + if (output / 'bundle.json').exists(): + raise FileExistsError('Use a new bundle directory to change a frozen study') + pins = {} + + def pin(relative): + pins[relative] = checksum(input_path(input_root, relative)) + + for entry in entries: + pin(entry['checkpoint']) + if entry.get('training'): + pin(entry['training']) + _, training_pins = validate_training(entry, input_root, manifest['data_root'], file_hashes=pins) + pins.update(training_pins) + for relative in entry.get('adapters', {}).values(): + pin(relative) + if entry.get('verification'): + pin(entry['verification']) + datasets, plans, corrections = {}, {}, {} + for name in dict.fromkeys(entry['dataset'] for entry in entries): + for split in ('valid', 'test'): + matching = [entry for entry in entries if entry['dataset'] == name] + policies = list(dict.fromkeys(inference_graph_for_split(name, split, entry.get('inference_graph')) for entry in matching)) + data = load_benchmark(input_path(input_root, manifest['data_root']), name, split=split, + inference_graph=policies[0]) + query_ids = {query.query: query.identity for query in data.queries} + key = dataset_key(dict(dataset=name, inference_graph=policies[0]), split) + datasets[key] = dict(context=data.context.identity, candidates=fingerprint(data.candidates), + metadata=data.metadata, queries=len(data.queries), + query_ids=fingerprint(sorted(query_ids.values()))) + for filename, digest in data.metadata['files'].items(): + relative = str(Path(manifest['data_root']) / dataset_spec(name)[1] / filename) + pins[relative] = digest + if split == 'test' and manifest.get('answer_filter') == 'corrected': + from dicee.query_answering.datasets import audit_plus_h_filters + audit = audit_plus_h_filters(input_path(input_root, manifest['data_root']), data) + relative = f'filters/{name}.json' + write_json(output / relative, audit) + corrections[name] = dict(path=relative, sha256=checksum(output / relative), per_shape=audit['per_shape']) + for filename, digest in audit['source_files'].items(): + pins[str(Path(manifest['data_root']) / dataset_spec(name)[1] / filename)] = digest + for policy in policies[1:]: + variant = load_benchmark(input_path(input_root, manifest['data_root']), name, split=split, + inference_graph=policy) + variant_key = dataset_key(dict(dataset=name, inference_graph=policy), split) + datasets[variant_key] = dict(datasets[key], context=variant.context.identity, metadata=variant.metadata) + for filename, digest in variant.metadata['files'].items(): + pins[str(Path(manifest['data_root']) / dataset_spec(name)[1] / filename)] = digest + prepared_plans = {} + for entry in entries: + if entry['dataset'] != name: + continue + layout = fingerprint({key: entry[key] for key in ('query_types', 'query_batch_size', 'query_order')}) + if layout not in prepared_plans: + plan = make_plan(data, dict(entry, _data_root=manifest['data_root']), input_root, query_ids) + relative = f'plans/{name}-{split}-{layout[:16]}.json' + write_json(output / relative, plan) + prepared_plans[layout] = dict(path=relative, sha256=checksum(output / relative)) + plans[f'{entry["id"]}/{split}'] = prepared_plans[layout] + bundle = dict(version=1, manifest=manifest, files=pins, datasets=datasets, plans=plans, filter_corrections=corrections, + source_sha256=code_identity(), implementation_sha256=implementation_fingerprint(), + created=time.time(), python=platform.python_version(), + protocol=dict(tie_policies=['sort', 'expected'], tie_equality='exact', + filtering=manifest.get('answer_filter', 'released'), + averaging='answers per query; queries per type; equal types', + observed_graphs=('entry-specific test graph; validation uses train; pinned by context hashes' + if all(e['dataset'] in PLUS_H_DATASETS for e in entries) else + 'published dataset-specific validation/test graphs; pinned by context hashes'), + candidates='dataset-specific; pinned by candidate hashes')) + write_json(output / 'bundle.json', bundle) + write_json(output / 'bundle.sha256.json', {'sha256': checksum(output / 'bundle.json')}) + return bundle + + +def verify_bundle(directory, input_root): + directory = Path(directory) + if checksum(directory / 'bundle.json') != read(directory / 'bundle.sha256.json')['sha256']: + raise ValueError('Bundle checksum mismatch') + bundle = read(directory / 'bundle.json') + if bundle['source_sha256'] != code_identity(): + raise ValueError('Source differs from the frozen bundle; freeze a new study') + for relative, digest in bundle['files'].items(): + if checksum(input_path(input_root, relative)) != digest: + raise ValueError(f'Input checksum mismatch: {relative}') + for plan in bundle['plans'].values(): + if checksum(directory / plan['path']) != plan['sha256']: + raise ValueError('Query plan checksum mismatch') + for correction in bundle.get('filter_corrections', {}).values(): + if checksum(directory / correction['path']) != correction['sha256']: + raise ValueError('Answer-filter correction checksum mismatch') + return bundle + + +def bundle_entry(bundle, entry_id): + """The frozen entry ``entry_id`` of ``bundle``.""" + matches = [entry for entry in bundle['manifest']['entries'] if entry['id'] == entry_id] + if len(matches) != 1: + raise ValueError(f'No frozen entry {entry_id!r} in this bundle') + return matches[0] + + +def run_job(bundle_dir, entry_id, input_root, output, *, device='cpu', phase='pilot', pilot_queries=2, + on_prediction=None, probe_only=False, on_control_prediction=None): + """Pilot validation or full test; never choose settings from test labels.""" + from dicee.models._inference import float32_precision_backends + bundle = verify_bundle(bundle_dir, input_root) + entry = bundle_entry(bundle, entry_id) + if phase not in ('pilot', 'test'): + raise ValueError('Choose the pilot or test phase') + if probe_only and phase != 'pilot': + raise ValueError('Probe selection is only available for validation pilots') + if phase == 'test': + validate_training(entry, input_root, bundle['manifest']['data_root'], final=True, file_hashes=bundle['files']) + if entry.get('blockers') or (entry['method'].endswith('-adapter') and not entry.get('finalized')): + raise ValueError(f'Entry is not ready for final testing: {entry.get("blockers", ["adapter recipe is provisional"])}') + if not entry.get('verification'): + raise ValueError('Final testing requires pinned reference parity evidence') + seed, threads = bundle['manifest']['seed'], bundle['manifest']['threads'] + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.set_num_threads(threads) + for backend in float32_precision_backends(): + backend.fp32_precision = 'ieee' + env = environment(device) + if phase == 'test': + evidence = read(input_path(input_root, entry['verification'])) + expected = dict(entry_sha256=fingerprint({key: value for key, value in entry.items() if key != 'verification'}), + source_sha256=bundle['source_sha256'], environment=env, + inputs_sha256=entry_inputs_identity(bundle, entry)) + if not evidence.get('passed') or any(evidence.get(key) != value for key, value in expected.items()): + raise ValueError('Parity evidence does not match this entry, source, and execution hardware') + output = Path(output) + identity = dict(bundle=checksum(Path(bundle_dir) / 'bundle.json'), entry=entry_id, phase=phase, + pilot_queries=pilot_queries if phase == 'pilot' else None, probe_only=probe_only, environment=env) + with BenchmarkCheckpoint(output / 'job', identity): + if (output / 'result.json').exists(): + result = read(output / 'result.json') + if result.get('benchmark_run', result.get('paper_run')) != identity: + raise ValueError('Result identity mismatch') + if checksum(output / 'ranks.sqlite3') != result['rank_trace_sha256']: + raise ValueError('Completed rank trace checksum mismatch') + for name, expected_checksum in result.get('comparison_results', {}).items(): + comparison = read(output / name / 'result.json') + if (checksum(output / name / 'result.json') != expected_checksum or + checksum(output / name / 'ranks.sqlite3') != comparison['rank_trace_sha256']): + raise ValueError('Completed comparison checksum mismatch') + return result + write_json(output / 'status.json', dict(state='loading', updated=time.time(), **identity)) + started = time.monotonic() + split = 'valid' if phase == 'pilot' else 'test' + data = load_benchmark(input_path(input_root, bundle['manifest']['data_root']), entry['dataset'], split=split, + inference_graph=entry.get('inference_graph')) + if data.context.identity != bundle['datasets'][dataset_key(entry, split)]['context']: + raise ValueError('Loaded inference graph differs from the frozen graph') + lookup = {query.identity: query for query in data.queries} + filter_corrections = None + if phase == 'test' and bundle['manifest'].get('answer_filter') == 'corrected': + from dicee.query_answering.datasets import corrected_answer_filters + correction = bundle['filter_corrections'][entry['dataset']] + filter_corrections = corrected_answer_filters(data, read(Path(bundle_dir) / correction['path'])) + query_lookup = {query.query: query for query in data.queries} + prediction_hook = (lambda query, scores: on_prediction(query_lookup[query], scores)) if on_prediction else None + control_hook = (lambda query, scores: on_control_prediction(query_lookup[query], scores)) if on_control_prediction else None + saved_plan = read(Path(bundle_dir) / bundle['plans'][f'{entry_id}/{split}']['path']) + plan = [lookup[key] for key in saved_plan['queries']] + ends = saved_plan['batch_ends'] + reference_batches = {} + if phase == 'pilot': + if type(pilot_queries) is not int or pilot_queries < 1: + raise ValueError('Positive pilot query budget required') + selected, selected_ends, counts, start = [], [], Counter(), 0 + for end in ends: + batch = plan[start:end] + if counts[batch[0].shape] < pilot_queries: + probes = batch[:pilot_queries - counts[batch[0].shape]] if probe_only else batch + reference_batches[probes[0].query] = [query.query for query in batch] + selected.extend(probes) + selected_ends.append(len(selected)) + counts[batch[0].shape] += len(probes) + start = end + plan, ends = selected, selected_ends + loading = time.monotonic() - started + if torch.device(device).type == 'cuda': + torch.cuda.reset_peak_memory_stats(device) + write_json(output / 'status.json', dict(state='evaluating', updated=time.time(), **identity)) + try: + recipe = {key: entry[key] for key in ('method', 'dataset', 'query_types', 'query_batch_size')} + recipe.update(checkpoint=str(input_path(input_root, entry['checkpoint'])), + root=str(input_path(input_root, bundle['manifest']['data_root'])), + options=dict(entry['options'])) + if 'inference_graph' in entry: + recipe['inference_graph'] = entry['inference_graph'] + if entry['method'] in REFERENCES: + recipe['vocabulary_dataset'] = entry['dataset'] + if 'per_shape' in recipe['options']: + recipe['per_shape'] = recipe['options'].pop('per_shape') + else: + recipe.update(adapters={name: str(input_path(input_root, path)) for name, path in entry['adapters'].items()}, + operators=entry['operators'], selection_protocol=entry['selection_protocol'], + adapter_ablation=entry.get('adapter_ablation', False)) + report = evaluate_method(recipe, output=output, data=data, split=split, device=device, seed=seed, + query_plan=plan, batch_ends=ends, rank_trace_path=output / 'ranks.sqlite3', provenance=identity, + query_order='published' if entry['method'] in REFERENCES else entry['query_order'], + write_result=False, on_prediction=prediction_hook, reference_batches=reference_batches, + on_control_prediction=control_hook if entry['method'] not in REFERENCES else None, + filter_corrections=filter_corrections) + profile = ('bounded' if not entry['options'].get('reference_batching', False) else 'reference') if entry['method'] in ('cqd', 'cqd-hybrid') else ('reference' if entry['method'] in REFERENCES else 'native') + common = dict(execution_profile=profile, reference=entry['reference'], + graph_ablation=entry.get('graph_ablation'), + graph_recipe_sha256=fingerprint(dict( + recipe={key: value for key, value in entry.items() + if key not in ('id', 'inference_graph', 'graph_ablation', 'verification', 'reference')}, + inputs=entry_inputs_identity(bundle, entry), source=bundle['source_sha256'], + filters=bundle.get('filter_corrections', {}).get(entry['dataset']), + plan=bundle['plans'][f'{entry_id}/{split}']['sha256'], seed=seed, environment=env)), coverage=dict( + selected_types=entry['query_types'], expected_types=data.metadata['expected_query_types'], + complete_benchmark_types=set(entry['query_types']) == set(data.metadata['expected_query_types']), + complete_16_types=set(entry['query_types']) == set(PLUS_H_SHAPES)), + dataset_loading_seconds=loading, total_wall_seconds=time.monotonic() - started, + peak_cuda_allocated_bytes=torch.cuda.max_memory_allocated(device) if torch.device(device).type == 'cuda' else None) + save_results(report, output, run=identity, metadata=common) + write_json(output / 'status.json', dict(state='complete', updated=time.time(), **identity)) + return report + except BaseException as error: + write_json(output / 'status.json', dict(state='failed', error=str(error), updated=time.time(), **identity)) + raise + + +def entry_inputs_identity(bundle, entry): + selected = {entry['checkpoint'], *entry.get('adapters', {}).values()} + if entry.get('training'): + selected.add(entry['training']) + prefix = str(Path(bundle['manifest']['data_root']) / dataset_spec(entry['dataset'])[1]) + '/' + return fingerprint({key: value for key, value in bundle['files'].items() if key in selected or key.startswith(prefix)}) + + +def integration_passed(path, bundle, entry): + """Whether ``path`` holds a passing KGFM integration check of this frozen entry, inputs and source.""" + if not Path(path).is_file(): + return False + evidence = read(path) + return bool(evidence.get('passed') and evidence.get('source_sha256') == bundle['source_sha256'] + and evidence.get('entry_sha256') == fingerprint({k: v for k, v in entry.items() if k != 'verification'}) + and evidence.get('inputs_sha256') == entry_inputs_identity(bundle, entry)) + + +def parity_passed(path, bundle, entry, reference, comparison_reference=None): + """Whether ``path`` holds passing parity evidence of this frozen entry, inputs and source against these oracles.""" + if not integration_passed(path, bundle, entry): + return False + evidence = read(path) + comparison = evidence.get('upstream_comparison', {}).get('reference_sha256') + return (evidence.get('reference_sha256') == checksum(reference) + and comparison == (checksum(comparison_reference) if comparison_reference is not None else None)) diff --git a/benchmarks/cqa/tests/__init__.py b/benchmarks/cqa/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/cqa/tests/recipe_fingerprints.json b/benchmarks/cqa/tests/recipe_fingerprints.json new file mode 100644 index 000000000..935ebdf45 --- /dev/null +++ b/benchmarks/cqa/tests/recipe_fingerprints.json @@ -0,0 +1,209 @@ +{ + "plus_h/baselines.json": { + "cone-FB15k237+H": "b777464a0561176a7797ad69c58bbc4f48b4f1f1531ac5c67dc8dbeef662a0a5", + "gnnqe-FB15k237+H": "91f82093f41dd43afc45e66c871648498fa4f9b827f36dc131fc186e160c52db", + "ultraquery-FB15k237+H": "7e61565765bc01f245d977095a0a91bfde7e22c228f63d6e6bdb00bafaecbe31", + "clmpt-FB15k237+H": "8594c53f58b3305715e5d45571b7ac4792209a603deacf405c779c8649f8a394", + "cqd-FB15k237+H": "2f8594e09e9aeaedfd4a530b85115f9029775bc3f3bd8b4f40a4a377374d663f", + "cqd-hybrid-FB15k237+H": "83dd457559834229bd99e518eaa3482f887d9d24fb908d0c08c51f348a95c1f9", + "qto-FB15k237+H": "e1d7c0c74e9f26a17e9650007d9e2750c5dc7fc6cdb54e05d17709a3d078769c", + "cone-NELL995+H": "c8b275487e2b7bd6a25af041b69c3f5bbe9089da718b7f5d26e0d223f7afd1c4", + "gnnqe-NELL995+H": "4e854b0b944fca240caedfd3db0994870354fa577be2146d9b7155d7d2107a78", + "ultraquery-NELL995+H": "8daf270d1dfef53ce684e513d7e825385ffc66165ff974792ed90efc44f0cec2", + "clmpt-NELL995+H": "1d69cdee8d2133eaf09286cd3134c934edee36d5580981d8ae77e20eb94b8bf9", + "cqd-NELL995+H": "435d309810afafc07fede359c0f53aeee8d6de6f3ea813dcfc7e69e347a04006", + "cqd-hybrid-NELL995+H": "9572fb778227a97087e871f0c6852240e582e9a41e8fcce83c340c9e485942ea", + "qto-NELL995+H": "cbc241f18bacddfe47d801f3aa36a69e0598df782db455469b13ebc832abf6a0", + "cone-ICEWS18+H": "13714f669fcfe488f65afe8cc18ce25a144f3e25b0b21bccdae403500dba0b74", + "gnnqe-ICEWS18+H": "62ac479f319b62ab29105b14e327dae0a303a6cf00d5f9989d38e135d833af52", + "ultraquery-ICEWS18+H": "c0cba93ed7bc90435d613a856b49dc1b22ff7aed523ef704204abd7fe4e48ca8", + "clmpt-ICEWS18+H": "2c0d83424f3b632f767c1fc8ee9bfc5a325e6de99d72ce1f22b2e28731d466a9", + "cqd-ICEWS18+H": "742156216a0ee43512138fad200af95e494b2146ae160b17c2e8f4839b893bda", + "cqd-hybrid-ICEWS18+H": "37e3d6d0307f3661c1a58eebf3f07960605008ae959b421ffc3fe63f98509a89", + "qto-ICEWS18+H": "980a9eff4f1feeadae6314974d8e44da51c51bbfe5479b9087c6cc78baf7d01e" + }, + "plus_h/kgfm_adapters.json": { + "ultra-product-intersections-FB15k237+H": "b69ee51b10efde939da3acca958737342f0b9c04cb87029a433f96a9694dbd09", + "ultra-product-intersections-NELL995+H": "f6a4a3f314e45764186125e90e21c7874be9bdf6903cf3dc27e1d31f244ddcb4", + "ultra-product-intersections-ICEWS18+H": "c6283b9ab962a54bd825c230173b16fc10e8a9529dab1c5733755a2770a937fb", + "trix-product-intersections-FB15k237+H": "6fabf9f473c59f4519446bfe53445cd8be1a7bbb0be889d88103c7c8281f1a94", + "trix-product-intersections-NELL995+H": "69cb115effd0ee6d9c14c06f8d4866e3ff2148260174b36722d0a7b68e8dab3c", + "trix-product-intersections-ICEWS18+H": "6f84bbe2e79bb47b5c8a788db48dd51079f52ad1274a839043fe472d36f66b37" + }, + "plus_h/kgfm_kgicl.json": { + "kgicl-product-intersections-FB15k237+H": "3e0c3f327646759f60036d9f5f09be48bc1693d483d38fe2fae8fca625cf6781", + "kgicl-product-intersections-NELL995+H": "9ac42998bb1d26ba9c06da2ff84f467274c5bfe7817d4aa6c6d77c5d69cfb8e4", + "kgicl-product-intersections-ICEWS18+H": "5977665fc7a15759ee0c2a40a1b71b757bdd222a52ff551175c13d7fc65fc146" + }, + "ultraquery/baselines.json": { + "ultraquery-FB15k237LogicalQuery": "529aa658841bff87867a50b626cae7e9c01dc8a7b38b5177eda2af712988e66d", + "ultraquery-FB15kLogicalQuery": "01ae98208647999c884181ef7308944ffc47555d260b4f27d7bdf2bc0dea5f7f", + "ultraquery-NELL995LogicalQuery": "16df73e62cb45ee294b26c5b683050d74253cb8dbc2a4a1b9cd6797eba7cede9", + "ultraquery-InductiveFB15k237Query:106": "c5073b00330b53eacd1a8915c0bce42b9bd3529a58e594289a461ef4449be596", + "ultraquery-InductiveFB15k237Query:113": "6889d15aa1354a9b09f03437cf2cfcb36693f04a381e2bc4ff330fcb4988c105", + "ultraquery-InductiveFB15k237Query:122": "aa367e591c621ac797c68e88b4301fe6fb2bb02dac24325772ebd12547e42a7d", + "ultraquery-InductiveFB15k237Query:134": "74e4dc629d34402184e676ee1ffb3201e74e0210de3d3fc48a8717e23e66653f", + "ultraquery-InductiveFB15k237Query:150": "68be6b843432b955d14fe9843fff09f211bf5bbc7df0a5722798600327b24fac", + "ultraquery-InductiveFB15k237Query:175": "26fb1d46a553d085106dd0d83548f8307eb4af270c639bffb390a96b164c8831", + "ultraquery-InductiveFB15k237Query:217": "3ba266366574e77f74e9427c358940025fe6a7a737d616af64f8e6abdcac173b", + "ultraquery-InductiveFB15k237Query:300": "137fa330abe62fcc62243e7cdce5a94d677e37cdf188fcc29bb9a713100a253f", + "ultraquery-InductiveFB15k237Query:550": "4cdadd990e7e369ce5771a82070d3d3927a702ead0fe561c49bb26ab6a26c0cf", + "ultraquery-WikiTopicsQuery:art": "9f167ee40580fb4e03957e9e3580204c05b1c3b5be300528d7a7b55b8ddc21f9", + "ultraquery-WikiTopicsQuery:award": "b371b7eb035887b1adbf674bf554660b1cc871d18fbfb523987e6c3c2a586ec2", + "ultraquery-WikiTopicsQuery:edu": "e49caaab3d330ac85b5e9e0a7fb5b2075237d0f6504c3cf380f444725b40fb6e", + "ultraquery-WikiTopicsQuery:health": "b68f71dab3d57c6552b1bc39ddf90b5d883182cb592f76fd79cdd13d23ac056a", + "ultraquery-WikiTopicsQuery:infra": "aebc9bb5dfadc082387fea7bda8cdcab41c0cc37063005fe11296dbf2c5d527f", + "ultraquery-WikiTopicsQuery:loc": "bb4317cddf9490ed94bb88d7beb70507b2962256812eecd3a59cfc716c50b652", + "ultraquery-WikiTopicsQuery:org": "92a78b53b6dd1d670978c482d2cd4d2ae9f5289985847a5937598d28a60cc7e0", + "ultraquery-WikiTopicsQuery:people": "6e8ad2618a2f1ba0c4c4d4e7a052c90319a0587cd4c2e97f8fb3daeb13027ef4", + "ultraquery-WikiTopicsQuery:sci": "89a3adb5346ad51c3c8e245953755d54455d04d5b55c496e14f6e5b275c3d1aa", + "ultraquery-WikiTopicsQuery:sport": "40615d4debf7ef280ecb1386d8605741b73203e16b4b78f9c4decf0ac9173cd3", + "ultraquery-WikiTopicsQuery:tax": "b6ad26d90fced664325931c4e938e41cb1dbf9ebc43323a09bee9e7b8a403c75" + }, + "ultraquery/comparisons.json": { + "ultraquery-lp-FB15k237LogicalQuery": "dbfdd8496fff9d398ea92815edae6c2eab3fdf1e08175f9edd040297a8baced4", + "ultraquery-lp-no-threshold-FB15k237LogicalQuery": "a963727dc2154cb1636b61644805055ffd248abe946bd6d8a0febbb53c72eba1", + "ultraquery-lp-FB15kLogicalQuery": "2942f971df1a2a38a2afdc9896968748f12af2fce65d402d8ad5dd2bb02c215d", + "ultraquery-lp-no-threshold-FB15kLogicalQuery": "eabcdea0bfb9b93cbba21ef8a8f61d3dee33c673417be502182e9a61a076fe17", + "ultraquery-lp-NELL995LogicalQuery": "4fb015b5b6a1d862fd4d8ab6060aa5e8f26d270620491d0fd3cee11f70391335", + "ultraquery-lp-no-threshold-NELL995LogicalQuery": "a0cff7ce323c74c231a7862a36295d6d2fb69d5552ca7253156d2a4657d2ff40", + "ultraquery-lp-InductiveFB15k237Query:106": "d7517583ccd0462ccce2e51601844b60c2999a91add248389c421834ea431169", + "ultraquery-lp-no-threshold-InductiveFB15k237Query:106": "5595ffcc392e0701b9b57c9a481b0463f6de9e62e81613e7506e29b7a718b2d7", + "ultraquery-lp-InductiveFB15k237Query:113": "060f0d6218ddd8ddd3d88c5961dcb4687b02f8b492d3c1c5cadccff415b03668", + "ultraquery-lp-no-threshold-InductiveFB15k237Query:113": "439641456aeea80e1dc8fcc14707f9e374830e5a49f1ee3fd7619d1908bb68bb", + "ultraquery-lp-InductiveFB15k237Query:122": "a9f55a1f18bdea9e93d2d50ca3eaa794d29f21de0f5c9e9cc1303a2eb4b3abd2", + "ultraquery-lp-no-threshold-InductiveFB15k237Query:122": "bb183066fe67fc2257c6e29294e9be67fb9607d1bf1829c7bd6875257de55b6a", + "ultraquery-lp-InductiveFB15k237Query:134": "24bf68250d0d3a17ca2844089b58fb0c536414e87b4fffc35fe93252804a0cae", + "ultraquery-lp-no-threshold-InductiveFB15k237Query:134": "a44dea55c40f3d9ec3173a142fe61e7fada997e65563863f47c991fd94e4640f", + "ultraquery-lp-InductiveFB15k237Query:150": "fbb584c8ef2d7c1b5a1326a7d7f4d22965d2451cc4734f985574a905328e2461", + "ultraquery-lp-no-threshold-InductiveFB15k237Query:150": "019325f11cba8d58ff67b3bd0b8befa6a993fd8274822858392acf5df4967c17", + "ultraquery-lp-InductiveFB15k237Query:175": "532a92102320abedbf9b0b7e4521227b44f9f269c48da49ec7f8c0fea1f37291", + "ultraquery-lp-no-threshold-InductiveFB15k237Query:175": "dd067eaa46b768cbdd2fa8ec5439e0ecbc9dea8542733326cd8a3b4ce23c665f", + "ultraquery-lp-InductiveFB15k237Query:217": "ea4323a26c441c05b1b5ff3bb46ed2ee911a86d5ab193cfe644d8093b9458d60", + "ultraquery-lp-no-threshold-InductiveFB15k237Query:217": "302711780fddcf86b0bdfb40de34b936182e029429a8fdadefb59779fbad289c", + "ultraquery-lp-InductiveFB15k237Query:300": "a57db5a0229a0092b93ef8d46e667dcc55173ed61b72d0aca9b122f95a531de9", + "ultraquery-lp-no-threshold-InductiveFB15k237Query:300": "74c6e9bb759ea2df0089ec6aaa14c82eb50b466338f1255f7d8cb2450b7d7dbe", + "ultraquery-lp-InductiveFB15k237Query:550": "bfd49146b7bba02f9248483d7671d74a0d11c62865c47defe3657a631a4d8239", + "ultraquery-lp-no-threshold-InductiveFB15k237Query:550": "d3bc8fc45c74056dc974ab85e0c2625482f2ac98fc9371a72e1eafb1d1ad60af", + "ultraquery-lp-WikiTopicsQuery:art": "469b6f3a77c4773e910e76759a4e7612de4fb90c01333ea9ff51d4774e81a65e", + "ultraquery-lp-no-threshold-WikiTopicsQuery:art": "f10e58cf5be73c2325641aa71c476185884697610475ac1f19cbb77a985ed08a", + "incoming-relation-WikiTopicsQuery:art": "f84fe5bae3e9370e1f2372d05e18f0947780e65fe882297e96ed8692c895866d", + "ultraquery-lp-WikiTopicsQuery:award": "dc51e75a10149557cd3327fd4fd9ddd7dda8579faa8e3cea060c3f53813734c8", + "ultraquery-lp-no-threshold-WikiTopicsQuery:award": "7f8a1b01c3274abba4a8330b530cedfcf55fc803ca5d933e61298b2ed7c15d09", + "incoming-relation-WikiTopicsQuery:award": "89c89da1bda6b88d8274450f7a59272fcce3215af6ac2c8efe9c67d91e0a870b", + "ultraquery-lp-WikiTopicsQuery:edu": "392b620f3207eda26be33bdbd78954cac8046616941ad066d88edee1d2b2ce72", + "ultraquery-lp-no-threshold-WikiTopicsQuery:edu": "def2bb596f29b1382397c1ccc6a288632376843db132c3141a3d0c98f4996200", + "incoming-relation-WikiTopicsQuery:edu": "4450dbdb181c8e0a95d321381d80f4564a190a78284e31e909df145c9a8da09e", + "ultraquery-lp-WikiTopicsQuery:health": "9035ad431fd94baaa485b669e5d866ca17d55e0361ae9279e69bf6583d2bc7f4", + "ultraquery-lp-no-threshold-WikiTopicsQuery:health": "d1d75775f4d019b7ea95eef3f8e5c1327b571b8007717153a5076b5a7dafb710", + "incoming-relation-WikiTopicsQuery:health": "3d4de5790d6e115a70aa2def0d9bca9f43958e7ecb06ffb147ee06f1d3c3f273", + "ultraquery-lp-WikiTopicsQuery:infra": "f77c293a5aa2f24cb7c2a3c1df75b4dd2170856c4e8ccc3e26b5250b3d9b423e", + "ultraquery-lp-no-threshold-WikiTopicsQuery:infra": "595ee1f9f6667b4c99480f481a7bef440195b40ef06a3bcc6b2aa18a9f33c59f", + "incoming-relation-WikiTopicsQuery:infra": "d0579e7faf5ecdddf7daa9f0ff9d1a26530bd5eb99a94f8b1d55eacc4df483b3", + "ultraquery-lp-WikiTopicsQuery:loc": "9ca9886f6ba850f0a33ed341df262c53a3f51dfc039afb5bb7f1250fbccede91", + "ultraquery-lp-no-threshold-WikiTopicsQuery:loc": "8ab7c739c5ece4ae216fb20f3398c5904035ce0915cfd9efafd71a3660845fba", + "incoming-relation-WikiTopicsQuery:loc": "1aa09be175cd1f1e50b2122a50bc14f41c6f78ffc2f5c0d327a1493e33a145ae", + "ultraquery-lp-WikiTopicsQuery:org": "4c6bb425601958397b13325a3d5be8b9966da03d8960a4194214140f033fa4d3", + "ultraquery-lp-no-threshold-WikiTopicsQuery:org": "ba0eb0c8e4c6b20f0693a5414fd71747515a186ad73bebe572e1b72bad8057c8", + "incoming-relation-WikiTopicsQuery:org": "f800373bf4fbde6389b17089f5ee39f53171aa0fcd605314796d465b60fa4175", + "ultraquery-lp-WikiTopicsQuery:people": "7b4f924c501bdb6e35a80a372fd9cbe9bce471d93d71e9a15f33743493371540", + "ultraquery-lp-no-threshold-WikiTopicsQuery:people": "5ba0c5b20189a04577f591ff25af015a3fc60e04bae0024bf43276f7b8fd921e", + "incoming-relation-WikiTopicsQuery:people": "99db30b400072675395912609cc3a3f0f470d5ace0f93140ef4d7a76ab12d09f", + "ultraquery-lp-WikiTopicsQuery:sci": "dfbb20e0298ef0f8c3b3f5b19ecf3694247451c29697c62efd3ec04fa8e1293c", + "ultraquery-lp-no-threshold-WikiTopicsQuery:sci": "aab5467d9d5d09585726043a512720f48b993d0e8c60f085e1480c49665eeecb", + "incoming-relation-WikiTopicsQuery:sci": "3c16464e475c51daa2b38854ea1856dd00e0184b406ab057ab5f40b64b3b3e57", + "ultraquery-lp-WikiTopicsQuery:sport": "051a0ad562e33e6f2d2889369bc54d1bd94deef33ed57cc9dd0d3532918e9931", + "ultraquery-lp-no-threshold-WikiTopicsQuery:sport": "bd1bfbea70a7b7c60ccb6a69045bfd4b79d808b56694bf6f5b06d1c41324dcb3", + "incoming-relation-WikiTopicsQuery:sport": "af14d509a098ee3eb5b1a1dfaa7cd40b3e0803eeaa52158480b61671840b251e", + "ultraquery-lp-WikiTopicsQuery:tax": "2a3baeefe692b9938f53447f82e53c15b4a1be187462bd49928190d624881a95", + "ultraquery-lp-no-threshold-WikiTopicsQuery:tax": "150b9d100def6f28df7ec8c060e050dfc506e50b959f8c91d927cec7cc794f5f", + "incoming-relation-WikiTopicsQuery:tax": "92940ef98b13543f54d4e43f765e51d35aab803ab4e122b570bd27ed7a373433", + "qto-FB15k237LogicalQuery": "6d0c84fe3626ad70e8237e8d373193ed22311262aac96ba7c8b09770ab75ccd5", + "qto-NELL995LogicalQuery": "472be331538648555c38e58b7a02c6a0b9d0ac1710bc14686210bcf9d434e879" + }, + "ultraquery/kgfm_adapters.json": { + "ultra-product-intersections-FB15k237LogicalQuery": "287c4c1e7f5000232b37686fb2cd11d480b8963226c9e723396a6eac6c5d5e48", + "ultra-product-intersections-FB15kLogicalQuery": "eb453b1cb918f07548d613970c35d12bf9cd7358acf5bc9fd18fbc8e7da45536", + "ultra-product-intersections-NELL995LogicalQuery": "29040aa44a675f7bae08f50f11eafff27fbfd6624e1fca5be03999ae2427f887", + "ultra-product-intersections-InductiveFB15k237Query:106": "7daf756989b77b0533b6fb557ca71b0b3a51a1031b63cc5b98128719209222de", + "ultra-product-intersections-InductiveFB15k237Query:113": "81552189926526ed9e2bdf8fe87ab52f0694322cb346668f190eb2bb60bf0901", + "ultra-product-intersections-InductiveFB15k237Query:122": "159166cb777fd904b4bc48baa48e980925c841a657b5588699382cadbecb90c5", + "ultra-product-intersections-InductiveFB15k237Query:134": "b2e5c879638b717de49dc4aaf215a3d1f6e7b29511f21f23047e3052404e75c8", + "ultra-product-intersections-InductiveFB15k237Query:150": "e3fce4a47cfe2b60d731accaaa05cff15dc5935fb44d235a8dcdfc86dbe46077", + "ultra-product-intersections-InductiveFB15k237Query:175": "6af28e4a40732fbe77ed918b78eee663f1bdc3d85fd1468bdfe14b4d36e1e26c", + "ultra-product-intersections-InductiveFB15k237Query:217": "98f3277b203311f6a8f43d849adf1b785a4edf0e2369e88afbc088e937c54761", + "ultra-product-intersections-InductiveFB15k237Query:300": "8663e0173bba7a2d68f4ab0dccfc8953d901d100e4645a138bc80328208271dd", + "ultra-product-intersections-InductiveFB15k237Query:550": "cd242c77612ea35b082694b3acc37470afaa13b79c392a4890ffa8ada3092007", + "ultra-product-intersections-WikiTopicsQuery:art": "a2a4c3998c2b70d96f76fa5c208a7572aa1d003f537245ddba7805293d995728", + "ultra-product-intersections-WikiTopicsQuery:award": "41fd60ec92e9bfca93e371f49a50523079bdbd797140ee547beb989e5b5000c4", + "ultra-product-intersections-WikiTopicsQuery:edu": "f184d870d32e8b9ba212d30674e52a4d5083d4546f087df9b550b0f494a909e7", + "ultra-product-intersections-WikiTopicsQuery:health": "e0860965b3fe6efba4ed5e8d26a3812388c14e3853e483f28df0074871f8e2ed", + "ultra-product-intersections-WikiTopicsQuery:infra": "34d6d916257068bc41d74f101b0eb623fb0cde18d63f12d26d82cd7f3f8a0ee5", + "ultra-product-intersections-WikiTopicsQuery:loc": "46db49317b65728e40e2ebd1eda4489eceda6ced547e1e1878b0839b392112e3", + "ultra-product-intersections-WikiTopicsQuery:org": "1a0eaf3df270714b6e8bd7a7da6db50a55772ed9dbcb474f526c7d7caa5b5ede", + "ultra-product-intersections-WikiTopicsQuery:people": "c24dc427aaa681a3877510c45bd87772e9ed1003183dd49579225ee6b2819d29", + "ultra-product-intersections-WikiTopicsQuery:sci": "0305cb5e98f62f9022b3e558c411b65c2b316d6bd8fa626fa616e6a9aea576fc", + "ultra-product-intersections-WikiTopicsQuery:sport": "d6000c937fff7b281157a1afe8de25d0d9011febc3834fb7964f8a2e4f7969e5", + "ultra-product-intersections-WikiTopicsQuery:tax": "cc0dcdba8ea327700aa81c87620c1de31590f01c54b064002ef9f53465f48da5", + "trix-product-intersections-FB15k237LogicalQuery": "1be3cf045d6511e9fa27d588a192bf615efcdf285bd859eca74c2ee8b2d6a90e", + "trix-product-intersections-FB15kLogicalQuery": "978d9dc4efd48c154cc8dafcddf2bd6056a9cfc36dd9206facaff356e7fb5452", + "trix-product-intersections-NELL995LogicalQuery": "c0edaec7b91da7684f35f0bf5225d3d95ac3dd043fa89f6b288a6c153ce88838", + "trix-product-intersections-InductiveFB15k237Query:106": "5f304ec689a95e64cbe2d78a36fdd1b16451f510ec60de87169291b9e5ed1888", + "trix-product-intersections-InductiveFB15k237Query:113": "81b0daebd89d258a16bfdbb4204f0e81647d283fe0ad0dd7594fdd8b1914798b", + "trix-product-intersections-InductiveFB15k237Query:122": "e1922ac49ff1fa062ad7e69dc5d4e62a3ce6706c0e2229892d2594e85bf13240", + "trix-product-intersections-InductiveFB15k237Query:134": "b24a1a635170150791fd01a5bf84556bdd8c8947cbb1d26207c35c70ae8021ae", + "trix-product-intersections-InductiveFB15k237Query:150": "0d31a07a19fa186d58d8aee2a1c778ec88195732340907afd557112a710e90c1", + "trix-product-intersections-InductiveFB15k237Query:175": "3419ce50681b12b8243ed05fbc779b6150453400d40dc520961e324a5f975088", + "trix-product-intersections-InductiveFB15k237Query:217": "7224a0bb0c9c2cfb778181cc6942ded084d1458ca12cc462920be438b8f29e1e", + "trix-product-intersections-InductiveFB15k237Query:300": "3c4c21948c52c4ffb10f462984279b425671dfd9088ae3b6864313c99a7edbba", + "trix-product-intersections-InductiveFB15k237Query:550": "0721fae25c558915cdcc154c6563c57d0fa933b5f30b0e9604e7308bd435ced5", + "trix-product-intersections-WikiTopicsQuery:art": "24620fd2cf94976a906b063418aebfbea7bff4219ec86e574437643bfffd967c", + "trix-product-intersections-WikiTopicsQuery:award": "d0f497a6a7dc1035fdcfb9a0b2b699aa5033bc235e0f1f34838bd8a18115131c", + "trix-product-intersections-WikiTopicsQuery:edu": "79a42ccaf3fee6e6aa8e5f1669301cf9e0b45c8895f3ecb01227f297f9b67891", + "trix-product-intersections-WikiTopicsQuery:health": "7f785b8624f72b7028dbd83eedd1b436a48de90364313089f345a9212c55ac8e", + "trix-product-intersections-WikiTopicsQuery:infra": "52920e879c823801319a7bc994eff5557da7171311298465f41e4faf61222599", + "trix-product-intersections-WikiTopicsQuery:loc": "3daa68f827ca911ef0501a02ba42811b985b9ee0f9ee29b372769c40a369d4ae", + "trix-product-intersections-WikiTopicsQuery:org": "106df045f26443278f54aa129a0ac2ae1e56a1e7516efbec6d5a9cc11845046b", + "trix-product-intersections-WikiTopicsQuery:people": "a07fa82767dabbe08c71c767b551da571ad930f210c22502675ce476bf8d907f", + "trix-product-intersections-WikiTopicsQuery:sci": "64a1ed5aadb56eac0a836cf421fb0f3ecb3db6580e2f6d2e0117a2e199449fbf", + "trix-product-intersections-WikiTopicsQuery:sport": "789c3e69c42d240f41d9b14cc72dee70f887a0498b14de7b92e50b89975b0c69", + "trix-product-intersections-WikiTopicsQuery:tax": "ca14cbcb7938c80e89c121d75f71c9163c26cc6dc622efabb7e35ebc3457dd40" + }, + "ultraquery/kgfm_kgicl.json": { + "kgicl-product-intersections-FB15k237LogicalQuery": "cff9eea715c7a4a4768486e720d7c0ecbb7d5a35354d0845f30b918bd3adc65c", + "kgicl-product-intersections-FB15kLogicalQuery": "a3c9e24991f977f0b94f058b9c526d2a019bbed8024487d12adf880a88fef751", + "kgicl-product-intersections-NELL995LogicalQuery": "b19316ac77ca1e865c9a309bd3f2be16df59763e150eaa77059091c16d2a90ea", + "kgicl-product-intersections-InductiveFB15k237Query:106": "e61fb816352242fc91e7dfc64e0bdc025aff3cffc722c522d5e347660b61b557", + "kgicl-product-intersections-InductiveFB15k237Query:113": "2d57a3d39d734335db012f43d436bcae09adb54b0710a2c70dc6f874566ad41d", + "kgicl-product-intersections-InductiveFB15k237Query:122": "d4abc1950f6d50f5a556a40244ed2ee360f9a255cca68cb1abbf8f8d0297d205", + "kgicl-product-intersections-InductiveFB15k237Query:134": "0c889ef74caa71112580914fcdda71bd238579fa8925d0ceb84e71d37727d76d", + "kgicl-product-intersections-InductiveFB15k237Query:150": "5de5b5c6a3ad1613cab2f3354a54a6cfa1887f1059d4ee39a3d50de43e6447bc", + "kgicl-product-intersections-InductiveFB15k237Query:175": "ad5f2c720057f394097d108506139d736ad8b416e569e04d11c2e0e17b6b66ee", + "kgicl-product-intersections-InductiveFB15k237Query:217": "e28d1b34573eaee7d94993e96ce9430a9e2386f1f5d538501d498048bfed3d07", + "kgicl-product-intersections-InductiveFB15k237Query:300": "86fffdd3c4358c56533596fb9a2b478f57582d84f70943c8b024fd77f2b6fed9", + "kgicl-product-intersections-InductiveFB15k237Query:550": "ac49339a01fee731825b09d78472150552470507adcb716e43995cb98372125a", + "kgicl-product-intersections-WikiTopicsQuery:art": "d4b346dad1f3e596d48d126c7791cb5194d1a0b3e4ae776c5df612633d276532", + "kgicl-product-intersections-WikiTopicsQuery:award": "3900bfe021dc7c9dc7db621eb913e505fdcd7a8a1a153da305bbecc1132c1fba", + "kgicl-product-intersections-WikiTopicsQuery:edu": "a81509b725b885d475a9b613969a16142aa69e9af8d62da310c7139944b028a2", + "kgicl-product-intersections-WikiTopicsQuery:health": "892b8215293c912e5de38089ea4f3ec0e6f6d41a3f512e67221d0e8b202217c0", + "kgicl-product-intersections-WikiTopicsQuery:infra": "43c4a124071b007bd0a63bbfb473273c824fc395a9955cf6792c1a8fe41209be", + "kgicl-product-intersections-WikiTopicsQuery:loc": "87095e5bd34466c4a457233bb95ea552c4059a9de22d2596b9837ea2f9cfb2eb", + "kgicl-product-intersections-WikiTopicsQuery:org": "06382ab313032eab6443a8bf6a8ca7d714454dbd3768a221cd3d55c8503fd78e", + "kgicl-product-intersections-WikiTopicsQuery:people": "185d92a96b7679d19fd8388e20e5790f20ba5fee09d02dcdf8d954d66413a69e", + "kgicl-product-intersections-WikiTopicsQuery:sci": "8f26bbc1b3a2fe0fd764a0fb6958e4bff8566907751d253728f4e380836ad288", + "kgicl-product-intersections-WikiTopicsQuery:sport": "7a8d96f2a063eb4d2ee0b68ddcf8d2aac1f15870bf662413eb834ec0a423997c", + "kgicl-product-intersections-WikiTopicsQuery:tax": "dd2b9ccb6fe96f624635c3d1693204bfe939942d41b50413635a30657873c6bb" + }, + "ultraquery/trained_baselines.json": { + "qto-FB15kLogicalQuery": "fbfdca9fe1cbcf143a8d8e2db1f3df8e341454b21b5325e7abe6b92de9be8b48", + "inductive-gnnqe-InductiveFB15k237Query:106": "1fa57c57fae80e52dcde4de9dbcca48dc6dcd8b22639792aa2478d79daf530bd", + "inductive-gnnqe-InductiveFB15k237Query:113": "b77cbaa878b8ae58e5bd878a09df7c1d8a992eb8778ebb6deade5bf55a43845f", + "inductive-gnnqe-InductiveFB15k237Query:122": "149f13b82a1fb5466de8299340222b1bd6eab900a57d8ecee708b70ff3c76c6c", + "inductive-gnnqe-InductiveFB15k237Query:134": "97de3fe8f047d42b6ddad861ba4b809a384cece028e7074cda7177684dfc8c94", + "inductive-gnnqe-InductiveFB15k237Query:150": "2609eacb22e59fc0298f171fee0e4719c69fc371378093d3995b8bab873ae4fc", + "inductive-gnnqe-InductiveFB15k237Query:175": "72f4fcf6ce5744c505c51f0287d298c0e8fe3f5caf00037cf61e34e36b51bc6a", + "inductive-gnnqe-InductiveFB15k237Query:217": "b5e9faa1ec482b963b9e4aac10da6fc45d7aa107711359bf5142f68c2b7c1782", + "inductive-gnnqe-InductiveFB15k237Query:300": "e32809e684d3ce1e0adc5f1d07c123fb9223d84850ca7f1e284e1d8ebf75e234", + "inductive-gnnqe-InductiveFB15k237Query:550": "817f41685b53ecb8cbea82fb0640c79c12dccb70f23beda6dbdfade138951511" + } +} diff --git a/benchmarks/cqa/tests/test_adapter_pairs.py b/benchmarks/cqa/tests/test_adapter_pairs.py new file mode 100644 index 000000000..a4739c31d --- /dev/null +++ b/benchmarks/cqa/tests/test_adapter_pairs.py @@ -0,0 +1,369 @@ +"""Paired calibration controls, shared logits, and crash-safe comparison traces.""" + +import subprocess +import sys + +import pytest +import torch + +from benchmarks.cqa import cli +from benchmarks.cqa.manifests import REPO, suite_directory +from benchmarks.cqa.reports import adapter_effects, export_reports, trace_queries +from benchmarks.cqa.study import freeze, integration_passed, read, run_job, verify_bundle, write_json +from benchmarks.cqa.tests.test_protocol import paper_fixture, small_data, use_ultraquery_fixture +from dicee.query_answering.benchmark import evaluate_benchmark +from dicee.query_answering.context import state_fingerprint +from dicee.query_answering.engine import AtomicBatchCache, AtomicScorer + + +class BatchRows(torch.nn.Module): + num_entities, num_relations = 6, 1 + + def __init__(self): + super().__init__() + self.values = torch.nn.Parameter(torch.arange(36.).reshape(6, 6)) + self.calls = 0 + + def forward_k_vs_all(self, conditions): + self.calls += 1 + return self.values[conditions[:, 0]] + len(conditions) / 100 + + +def test_shared_atomic_cache_preserves_batches_and_invalidates_weights(): + model = BatchRows().eval() + cache = AtomicBatchCache(48) + first = AtomicScorer(model, row_batch_size=2, raw_cache=cache) + second = AtomicScorer(model, row_batch_size=2, raw_cache=cache) + pair = [(0, 0), (1, 0)] + expected = first.rows(pair) + reused = second.rows(pair) + assert torch.equal(expected, reused) and model.calls == 1 and cache.hits == 2 + reused.zero_() + assert torch.equal(second.rows(pair), expected) + # A singleton has different floating-point batch semantics in this fixture. + assert not torch.equal(second.rows(pair[:1])[0], expected[0]) + assert model.calls == 2 and cache.used <= 48 + first.rows(pair) + assert model.calls == 3 # The complete pair was evicted. + with torch.no_grad(): + model.values.add_(1) + assert torch.equal(second.rows(pair), expected + 1) + assert model.calls == 4 + + +def test_row_cache_reuses_rows_across_batches_and_invalidates_weights(): + model = BatchRows().eval() + cache = AtomicBatchCache(10_000, granularity='row') + scorer = AtomicScorer(model, row_batch_size=2, raw_cache=cache) + first = scorer.rows([(0, 0), (1, 0)]) + assert model.calls == 1 and cache.computed == 2 + mixed = scorer.rows([(1, 0), (2, 0), (1, 0)]) + # Only the missing row is computed, in a batch of its own; the fixture marks batch sizes in the values. + assert model.calls == 2 and cache.computed == 3 and cache.hits == 2 + assert torch.equal(mixed[0], first[1]) and torch.equal(mixed[2], first[1]) + assert torch.equal(mixed[1], model.values[2].detach() + 1 / 100) + mixed.zero_() + assert torch.equal(scorer.rows([(0, 0)])[0], first[0]) and model.calls == 2 + with torch.no_grad(): + model.values.add_(1) + assert torch.equal(scorer.rows([(0, 0), (1, 0)]), first + 1) and model.calls == 3 + + +def test_cache_token_is_computed_once_per_evaluation(monkeypatch): + from dicee.query_answering import engine + calls = [] + original = engine.state_token + monkeypatch.setattr(engine, 'state_token', lambda model: calls.append(model) or original(model)) + scorer = AtomicScorer(BatchRows().eval(), row_batch_size=1, raw_cache=AtomicBatchCache(10_000, granularity='row')) + with scorer.evaluation(): + for head in range(3): + scorer.rows([(head, 0)]) + assert len(calls) == 1 + scorer.rows([(0, 0)]) + assert len(calls) == 2 + + +def test_paired_resume_replays_both_variants_and_preserves_metrics(tmp_path): + data = small_data() + + def learned(query): + return torch.tensor([10., 8., 7., 6., 5., 4.]) + + def control(query): + return torch.tensor([10., 1., 2., 3., 4., 5.]) + + settings = dict(checkpoint_dir=tmp_path / 'progress', checkpoint_identity={'paired': True}, + checkpoint_every=2, rank_trace_path=tmp_path / 'ranks.sqlite3', additional_tie_policies=('expected',)) + + def fail(query): + if query[0] == 3: + raise RuntimeError('interrupted between variants') + return control(query) + + with pytest.raises(RuntimeError, match='between variants'): + evaluate_benchmark(data, learned, comparison_predictors={'without-adapter': fail}, **settings) + seen = [] + + def resume(query): + seen.append(query[0]) + return learned(query) + + result = evaluate_benchmark(data, resume, comparison_predictors={'without-adapter': control}, **settings) + assert seen == [2, 3, 4, 5] + assert result['per_shape'] == evaluate_benchmark(data, learned)['per_shape'] + assert result['comparisons']['without-adapter']['per_shape'] == evaluate_benchmark(data, control)['per_shape'] + left, right = tmp_path / 'ranks.sqlite3', tmp_path / 'without-adapter/ranks.sqlite3' + assert len(list(trace_queries(left))) == len(list(trace_queries(right))) == 6 + effects = adapter_effects(left, right, bootstrap_samples=10) + assert effects['macro']['expected']['mrr'] == pytest.approx(.75) + assert effects['macro']['expected']['mrr_ci95'] == pytest.approx([.75, .75]) + assert adapter_effects(left, left, bootstrap_samples=0)['macro']['sort']['mrr'] == 0 + + +@pytest.mark.parametrize('backbone', ['ultra', 'trix', 'kgicl']) +@pytest.mark.parametrize('facts,ultraquery', [(None, False), ('none', False), ('atomic', False), ('both', False), ('both', True)]) +def test_paired_kgfm_verification_full_run_and_reports(tmp_path, backbone, facts, ultraquery): + from dicee.models import KGICL, TRIX, ULTRA + from dicee.query_answering.score_adapter import QueryScoreAdapter + manifest = paper_fixture(tmp_path) + if ultraquery: + use_ultraquery_fixture(manifest, tmp_path) + count = len(manifest['entries'][0]['query_types']) + model = {'ultra': ULTRA, 'trix': TRIX, 'kgicl': KGICL}[backbone](dict(num_entities=1, num_relations=1)) + # KG-ICL checkpoints keep their weights under 'state_dict'. + torch.save({'state_dict' if backbone == 'kgicl' else 'model': model.state_dict()}, tmp_path / 'kgfm.pt') + QueryScoreAdapter('context_scores', 1., bias_bound=8., + weights=torch.randn(2, 8, generator=torch.Generator().manual_seed(4)).double() / 5, + metadata={'backbone_state_sha256': state_fingerprint(model)}).save(tmp_path / 'adapter.json') + entry = manifest['entries'][0] + entry.update(id=f'{backbone}-paired', method=f'{backbone}-adapter', checkpoint='kgfm.pt', + options={'beam_size': 2, 'row_batch_size': 1, 'backend_batch_size': 1, 'cache_bytes': 4096, 'raw_cache_bytes': 4096}, + adapters={'product': 'adapter.json'}, operators={shape: 'product' for shape in entry['query_types']}, + selection_protocol='source-validation', finalized=True, adapter_ablation=True, + query_batch_size=4, query_order='relation') + if facts is not None: + entry['options']['observed_facts'] = facts + if ultraquery: + entry['options'].update(raw_cache_device='model', relation_cache_mb=1, projection_cache_mb=1) + freeze(manifest, tmp_path / 'bundle', tmp_path) + subprocess.run([sys.executable, '-m', 'benchmarks.cqa', 'plus_h', 'integration', '--bundle', str(tmp_path / 'bundle'), + '--input-root', str(tmp_path), '--entry', entry['id'], '--output', str(tmp_path / 'verification'), + '--device', 'cpu', '--probes', '1'], check=True, cwd=REPO) + evidence = read(tmp_path / 'verification/integration.json') + assert evidence['passed'] and len(evidence['queries']) == count * 2 + entry['verification'] = 'verification/integration.json' + freeze(manifest, tmp_path / 'verified-bundle', tmp_path) + result = run_job(tmp_path / 'verified-bundle', entry['id'], tmp_path, tmp_path / 'test', phase='test') + control = read(tmp_path / 'test/without-adapter/result.json') + assert result['queries'] == control['queries'] == count + assert result['inference']['cache_statistics']['backbone_reused_rows'] > 0 + assert control['paired_with'] == entry['id'] + assert control['inference']['calibration'] == 'without-adapter' + settings = result['inference']['observed_facts'] + assert settings == control['inference']['observed_facts'] + assert settings['atomic_mix'] == {'product': float(facts != 'none')} + assert settings['restore_answers'] == (facts in (None, 'both')) + assert read(tmp_path / 'adapter.json')['observed_mix'] == 1 + assert run_job(tmp_path / 'verified-bundle', entry['id'], tmp_path, tmp_path / 'test', phase='test') == result + rows = export_reports(tmp_path / 'test', suite_directory('plus_h') / 'published_results.json', tmp_path / 'reports', bootstrap_samples=10) + assert len(rows) == 2 and all(row['complete_test'] for row in rows) + assert all(row['complete_16_type_test'] == (not ultraquery) for row in rows) + assert all(row['observed_facts'] == settings for row in rows) + # Combined reports carry no result files, so the details keep the candidate and context identity. + details = {d['id']: d for d in read(tmp_path / 'reports/comparison.json')['details']} + for raw in (result, control): + detail = details[raw.get('benchmark_run', raw.get('paper_run'))['entry']] + assert detail['candidate_sha256'] and detail['context_sha256'] + assert {k: detail[k] for k in ('num_candidates', 'candidate_sha256', 'context_sha256')} == \ + {k: raw[k] for k in ('num_candidates', 'candidate_sha256', 'context_sha256')} + assert len(read(tmp_path / 'reports/adapter-effects.json')) == 1 + if ultraquery: + write_json(tmp_path / 'manifest.json', manifest) + args = ['ultraquery', 'evaluate', '--manifests', str(tmp_path / 'manifest.json'), '--input-root', str(tmp_path), + '--output', str(tmp_path / 'cli'), '--entries', entry['id'], '--split', 'test', '--device', 'cpu'] + cli.main(args) + direct = read(tmp_path / 'cli' / entry['id'] / 'result.json') + direct_control = read(tmp_path / 'cli' / entry['id'] / 'without-adapter/result.json') + assert direct['per_shape'] == result['per_shape'] + assert direct_control['per_shape'] == control['per_shape'] + cli.main(args) + assert read(tmp_path / 'cli' / entry['id'] / 'result.json')['per_shape'] == direct['per_shape'] + (tmp_path / 'test/without-adapter/result.json').write_text('{}') + with pytest.raises(ValueError, match='comparison checksum'): + run_job(tmp_path / 'verified-bundle', entry['id'], tmp_path, tmp_path / 'test', phase='test') + + +def test_verify_reuses_passing_integration_checks_and_rejects_partial_ones(tmp_path, monkeypatch, capsys): + from dicee.models import ULTRA + from dicee.query_answering.score_adapter import QueryScoreAdapter + manifest = paper_fixture(tmp_path) + model = ULTRA(dict(num_entities=1, num_relations=1)) + torch.save({'model': model.state_dict()}, tmp_path / 'kgfm.pt') + QueryScoreAdapter('context_scores', 1., bias_bound=8., weights=torch.full((2, 8), .1, dtype=torch.float64), + metadata={'backbone_state_sha256': state_fingerprint(model)}).save(tmp_path / 'adapter.json') + entry = manifest['entries'][0] + entry.update(id='ultra-paired', method='ultra-adapter', checkpoint='kgfm.pt', + options={'beam_size': 2, 'row_batch_size': 1, 'backend_batch_size': 1, 'cache_bytes': 4096, 'raw_cache_bytes': 4096}, + adapters={'product': 'adapter.json'}, operators={shape: 'product' for shape in entry['query_types']}, + selection_protocol='source-validation', finalized=True, adapter_ablation=True, + query_batch_size=4, query_order='relation') + manifest['entries'] = [entry] + study = tmp_path / 'study' + freeze(manifest, study / 'bundle', tmp_path) + target = study / 'verification' / entry['id'] + # The check verify would run, launched as a separate job. + subprocess.run([sys.executable, '-m', 'benchmarks.cqa', 'plus_h', 'integration', '--bundle', str(study / 'bundle'), + '--input-root', str(tmp_path), '--entry', entry['id'], '--output', str(target), '--device', 'cpu'], + check=True, cwd=REPO) + bundle = verify_bundle(study / 'bundle', tmp_path) + assert integration_passed(target / 'integration.json', bundle, entry) + assert not integration_passed(target / 'integration.json', bundle, dict(entry, options=dict(entry['options'], beam_size=3))) + args = ['plus_h', 'verify', '--input-root', str(tmp_path), '--output', str(study), '--device', 'cpu'] + (target / 'integration.json').rename(tmp_path / 'integration.json') + with pytest.raises(SystemExit): + cli.main(args) + assert 'KGFM verification requires a fresh directory' in capsys.readouterr().err + (tmp_path / 'integration.json').rename(target / 'integration.json') + monkeypatch.setattr(cli.subprocess, 'run', lambda command: pytest.fail('A passing check was run again')) + cli.main(args) + assert read(study / 'verified-manifest.json')['entries'][0]['verification'] == 'study/evidence/ultra-paired.json' + assert read(study / 'evidence/ultra-paired.json') == read(target / 'integration.json') + + +def test_graph_effects_pair_calibrations_and_reject_changed_recipes(tmp_path): + from dataclasses import replace + + from dicee.query_answering.method_evaluation import save_results + results = tmp_path / 'results' + for graph, scores in [('train', [10., 1., 2., 3., 4., 5.]), ('train+valid', [10., 8., 7., 6., 5., 4.])]: + data = replace(small_data(), name='FB15k237+H', split='test', + metadata=dict(inference_graph=graph, expected_query_types=['1p'])) + destination = results / graph + report = evaluate_benchmark(data, lambda query: torch.tensor(scores), + comparison_predictors={'without-adapter': lambda query: torch.zeros(6)}, + checkpoint_dir=destination / 'progress', checkpoint_identity={'fixture': graph}, + rank_trace_path=destination / 'ranks.sqlite3', additional_tie_policies=('expected',)) + report['inference'] = dict(method='ultra-adapter', calibration='learned') + report['comparisons']['without-adapter']['inference'] = dict(method='ultra-adapter', calibration='without-adapter') + save_results(report, destination, run=dict(entry=graph, phase='test'), + metadata=dict(reference={}, graph_ablation='paired', graph_recipe_sha256='same')) + export_reports(results, None, tmp_path / 'reports', bootstrap_samples=10) + effects = {row['calibration']: row for row in read(tmp_path / 'reports/graph-effects.json')} + assert len(read(tmp_path / 'reports/adapter-effects.json')) == 2 + assert effects['learned']['macro']['expected']['mrr'] == pytest.approx(.75) + assert effects['without-adapter']['macro']['expected']['mrr'] == 0 + target = results / 'train+valid/result.json' + value = read(target) + value['graph_recipe_sha256'] = 'changed' + write_json(target, value) + with pytest.raises(ValueError, match='Graph ablation changed'): + export_reports(results, None, tmp_path / 'invalid', bootstrap_samples=0) + + +def test_parallel_reports_are_byte_identical_to_sequential_reports(tmp_path): + from dataclasses import replace + + from dicee.query_answering.method_evaluation import save_results + + def scores(offset): + return lambda query: torch.rand(6, generator=torch.Generator().manual_seed(17 * query[0] + offset)) + + results = tmp_path / 'results' + for offset, graph in enumerate(('train', 'train+valid')): + data = replace(small_data(), name='FB15k237LogicalQuery', split='test', + metadata=dict(inference_graph=graph, expected_query_types=['1p'])) + destination = results / graph + report = evaluate_benchmark(data, scores(offset), comparison_predictors={'without-adapter': scores(5)}, + checkpoint_dir=destination / 'progress', checkpoint_identity={'fixture': graph}, + rank_trace_path=destination / 'ranks.sqlite3', additional_tie_policies=('expected',)) + report['inference'] = dict(method='ultra-adapter', calibration='learned') + report['comparisons']['without-adapter']['inference'] = dict(method='ultra-adapter', calibration='without-adapter') + save_results(report, destination, run=dict(entry=graph, phase='test'), + metadata=dict(reference={}, graph_ablation='paired', graph_recipe_sha256='same')) + export_reports(results, None, tmp_path / 'sequential', bootstrap_samples=50) + export_reports(results, None, tmp_path / 'parallel', bootstrap_samples=50, workers=3) + names = sorted(path.name for path in (tmp_path / 'sequential').iterdir()) + assert names == sorted(path.name for path in (tmp_path / 'parallel').iterdir()) + assert all((tmp_path / 'sequential' / name).read_bytes() == (tmp_path / 'parallel' / name).read_bytes() for name in names) + suite = read(tmp_path / 'parallel/suite-effects.json') + assert len(read(tmp_path / 'parallel/adapter-effects.json')) == 2 and len(suite) == 2 + low, high = suite[0]['macro']['sort']['all']['ci95'] + assert low < high + + +def test_filter_controls_share_predictions_and_resume_all_traces(tmp_path, monkeypatch): + from dataclasses import replace + + from dicee.query_answering._checkpoint import BenchmarkCheckpoint + from dicee.query_answering.method_evaluation import save_results + calls = [] + data = replace(small_data(), metadata={'expected_query_types': ['1p']}) + def predict(query): + calls.append(query) + return torch.tensor([10., 8., 7., 6., 5., 4.]) + options = dict(checkpoint_dir=tmp_path / 'progress', checkpoint_identity={'fixed': True}, + checkpoint_every=2, rank_trace_path=tmp_path / 'ranks.sqlite3', + additional_tie_policies=('expected',), comparison_predictors={'without-adapter': predict}, + filter_corrections={q.query: frozenset() for q in data.queries}) + original = BenchmarkCheckpoint.save + def fail_after_commit(self, state): + if state['completed'] == 4: + raise RuntimeError('interrupted filter control') + return original(self, state) + monkeypatch.setattr(BenchmarkCheckpoint, 'save', fail_after_commit) + with pytest.raises(RuntimeError, match='interrupted filter control'): + evaluate_benchmark(data, predict, **options) + monkeypatch.setattr(BenchmarkCheckpoint, 'save', original) + calls.clear() + report = evaluate_benchmark(data, predict, **options) + assert len(calls) == 8 # Four remaining queries, two predictors, both filters each. + assert report['averages']['all']['mrr'] == .5 + assert report['comparisons']['released-filters']['averages']['all']['mrr'] == 1. + for name, result in [(None, report), *report['comparisons'].items()]: + result['inference'] = dict(method='ultra-adapter', calibration='without-adapter' if name and name.startswith('without-adapter') else 'learned') + save_results(report, tmp_path, run={'entry': 'paired', 'phase': 'pilot'}, metadata={'reference': {}}) + rows = export_reports(tmp_path, None, tmp_path / 'reports', bootstrap_samples=10) + assert len(rows) == 4 + assert len(read(tmp_path / 'reports/adapter-effects.json')) == 2 + filters = read(tmp_path / 'reports/filter-effects.json') + assert len(filters) == 2 + assert all(row['macro']['expected']['mrr'] == -.5 for row in filters) + for name in ('', 'without-adapter', 'released-filters', 'without-adapter-released-filters'): + assert len(list(trace_queries(tmp_path / name / 'ranks.sqlite3'))) == 6 + options['filter_corrections'] = {} + with pytest.raises(ValueError, match='inputs/settings changed'): + evaluate_benchmark(data, predict, **options) + + +@pytest.mark.parametrize('backbone', ['ultra', 'trix', 'kgicl']) +def test_row_granular_raw_cache_reaches_the_engine_and_keeps_scores(tmp_path, backbone): + from dicee.models import KGICL, TRIX, ULTRA + from dicee.query_answering.score_adapter import QueryScoreAdapter + manifest = paper_fixture(tmp_path) + use_ultraquery_fixture(manifest, tmp_path) + model = {'ultra': ULTRA, 'trix': TRIX, 'kgicl': KGICL}[backbone](dict(num_entities=1, num_relations=1)) + # KG-ICL checkpoints keep their weights under 'state_dict'. + torch.save({'state_dict' if backbone == 'kgicl' else 'model': model.state_dict()}, tmp_path / 'kgfm.pt') + QueryScoreAdapter('context_scores', 1., bias_bound=8., + weights=torch.randn(2, 8, generator=torch.Generator().manual_seed(4)).double() / 5, + metadata={'backbone_state_sha256': state_fingerprint(model)}).save(tmp_path / 'adapter.json') + entry = manifest['entries'][0] + entry.update(id=f'{backbone}-paired', method=f'{backbone}-adapter', checkpoint='kgfm.pt', + options={'beam_size': 2, 'row_batch_size': 2, 'backend_batch_size': 2, 'cache_bytes': 4096, + 'raw_cache_bytes': 1 << 20, 'raw_cache_device': 'model', 'relation_cache_mb': 1, 'projection_cache_mb': 1}, + adapters={'product': 'adapter.json'}, operators={shape: 'product' for shape in entry['query_types']}, + selection_protocol='source-validation', finalized=True, adapter_ablation=True, + query_batch_size=4, query_order='relation') + results = {} + for granularity in ('batch', 'row'): + entry['options']['raw_cache_granularity'] = granularity + write_json(tmp_path / 'manifest.json', manifest) + cli.main(['ultraquery', 'evaluate', '--manifests', str(tmp_path / 'manifest.json'), '--input-root', str(tmp_path), + '--output', str(tmp_path / granularity), '--entries', entry['id'], '--split', 'test', '--device', 'cpu']) + results[granularity] = read(tmp_path / granularity / entry['id'] / 'result.json') + row, batch = results['row'], results['batch'] + assert row['inference']['options']['raw_cache_granularity'] == 'row' + for shape, values in batch['per_shape'].items(): + assert row['per_shape'][shape]['mrr'] == pytest.approx(values['mrr'], abs=1e-6) + statistics = row['inference']['cache_statistics'], batch['inference']['cache_statistics'] + assert statistics[0]['backbone_computed_rows'] <= statistics[1]['backbone_computed_rows'] diff --git a/benchmarks/cqa/tests/test_comparison_baselines.py b/benchmarks/cqa/tests/test_comparison_baselines.py new file mode 100644 index 000000000..e2361ba3b --- /dev/null +++ b/benchmarks/cqa/tests/test_comparison_baselines.py @@ -0,0 +1,279 @@ +"""Missing baseline integration, resume-safe heuristic and author training parity.""" + +import copy +import json +import os +import pickle +import subprocess +import sys +from pathlib import Path + +import pytest +import torch + +from benchmarks.cqa import cli +from benchmarks.cqa.evaluate import evaluate_manifest +from benchmarks.cqa.manifests import REPO, prepare_manifest, read_manifest, validate_entry +from benchmarks.cqa.oracles import verify_predictions +from benchmarks.cqa.study import freeze, run_job +from benchmarks.cqa.tests.test_ultraquery_launcher import PUBLIC, fixture +from benchmarks.cqa.train import verify_v2 +from dicee.query_answering import QueryContext, load_benchmark +from dicee.query_answering._checkpoint import checksum +from dicee.query_answering.methods import load_method +from dicee.query_answering.methods.heuristic import IncomingRelationHeuristic + +EXPORTER = REPO / 'benchmarks/cqa/verification/export_reference.py' + + +def test_comparison_recipes_cover_published_families(): + m = prepare_manifest([PUBLIC / 'comparisons.json', PUBLIC / 'trained_baselines.json'], + suite='ultraquery', answer_filter='released') + counts = {} + for e in m['entries']: + validate_entry(e) + counts[e['method']] = counts.get(e['method'], 0) + 1 + assert counts == {'ultraquery-lp': 46, 'incoming-relation': 11, 'qto': 3, 'inductive-gnnqe': 9} + thresholds = {e['dataset']: e['options']['threshold'] for e in m['entries'] + if e['method']=='ultraquery-lp' and '-no-threshold-' not in e['id']} + assert thresholds['NELL995LogicalQuery'] == .97 + assert all(value == (.97 if name=='NELL995LogicalQuery' else .8) for name,value in thresholds.items()) + assert all('2404.07198v2' in e['reference']['settings'] for e in m['entries'] if e['method']=='ultraquery-lp') + + +def test_heuristic_ignores_heads_and_random_order_is_subset_batch_resume_stable(): + context = QueryContext([(0, 0, 1), (2, 0, 3), (1, 1, 2)], 5, 2) + m = IncomingRelationHeuristic(context, seed=19) + a, b = (0, (0,)), (4, (0,)) + classes = m.membership_batch([('project', 0, ('anchor', 0)), ('project', 0, ('anchor', 4))]) + assert torch.equal(classes[0], classes[1]) + assert classes[0].tolist() == [0., 1., 0., 1., 0.] + scores = m.predict_batch([a, b]) + assert torch.equal(scores[0], m.predict(a)) and torch.equal(scores[1], m.predict(b)) + torch.rand(100) + assert torch.equal(scores, m.predict_batch([a, b])) + assert torch.equal(scores.flip(0), m.predict_batch([b, a])) + assert (scores[:, [1, 3]].min() > scores[:, [0, 2, 4]].max()) + assert not torch.equal(scores, IncomingRelationHeuristic(context, seed=20).predict_batch([a, b])) + + +def test_v2_gate_rejects_unknown_or_missing_training_inputs(tmp_path): + with pytest.raises(ValueError, match='Missing v2'): + verify_v2(tmp_path, '106') + (tmp_path / 'train_graph.txt').write_text('0 0 1\n') + with pytest.raises(ValueError): + verify_v2(tmp_path, '106') + + +def test_train_selection_cpu_and_no_writes_in_dry_run(tmp_path, capsys): + cli.main(['ultraquery', 'train', '--methods', 'inductive-gnnqe', '--datasets', 'InductiveFB15k237Query:106', + '--output', str(tmp_path / 'models'), '--dry-run']) + plan = json.loads(capsys.readouterr().out) + assert plan['device']=='cpu' and len(plan['jobs'])==1 + assert not (tmp_path / 'models').exists() + for suite, selection in (('ultraquery', ['--methods', 'qto', '--datasets', 'WikiTopicsQuery:art']), ('plus_h', ['--methods', 'qto'])): + with pytest.raises(SystemExit): + cli.main([suite, 'train', *selection, '--output', str(tmp_path), '--dry-run']) + + +def test_docker_train_defaults_to_cpu(tmp_path, capsys): + args = ['ultraquery', 'train', '--image', 'test', '--input-root', str(tmp_path), '--output', str(tmp_path / 'models'), + '--methods', 'qto', '--datasets', 'FB15kLogicalQuery', '--dry-run'] + cli.main(args) + out = capsys.readouterr().out + assert '--gpus' not in out and 'nvidia.com/gpu' not in out + assert 'test ultraquery train --input-root /inputs --output /results' in out + assert '--manifest-output-root /inputs/models' in out + with pytest.raises(SystemExit): + cli.main([*args, '--device', 'cuda']) + + +def trained_fixture(root): + manifest=fixture(root,'FB15kLogicalQuery') + entry=copy.deepcopy(next(e for e in read_manifest(PUBLIC/'trained_baselines.json')['entries'] + if e['method']=='qto')) + entry.update(checkpoint='model.pt',training='training.json') + manifest['entries']=[entry] + folder=root/'data/FB15k-betae' + (folder/'valid.txt').write_text('0\t0\t3\n') + (folder/'test.txt').write_text('1\t2\t3\n') + torch.save({'embeddings.0.weight':torch.zeros(8,16),'embeddings.1.weight':torch.zeros(4,16)},root/'model.pt') + record=dict(version=1,status='complete',method='qto',dataset=entry['dataset'],profile='smoke',dataset_release='fixture', + checkpoint_sha256=checksum(root/'model.pt'), + inputs={p.name:checksum(p) for p in folder.iterdir() if p.is_file()}) + (root/'training.json').write_text(json.dumps(record)) + return manifest,record,folder + + +@pytest.mark.parametrize('profile,release',[('smoke','fixture'),('fixture','fixture'),('custom','fixture')]) +def test_shipped_manifest_cannot_promote_smoke_or_fixture_weights(tmp_path,profile,release): + manifest,record,_=trained_fixture(tmp_path) + record.update(profile=profile,dataset_release=release) + (tmp_path/'training.json').write_text(json.dumps(record)) + assert manifest['entries'][0]['blockers']==[] + freeze(manifest,tmp_path/'bundle',tmp_path) + with pytest.raises(ValueError,match='smoke/fixture training checkpoint'): + run_job(tmp_path/'bundle',manifest['entries'][0]['id'],tmp_path,tmp_path/'final',phase='test') + pilot=run_job(tmp_path/'bundle',manifest['entries'][0]['id'],tmp_path,tmp_path/'pilot',phase='pilot') + assert pilot['queries']==14 + + +@pytest.mark.parametrize('changed_file',['id2ent.pkl','id2rel.pkl','train.txt','valid.txt','test.txt','test-hard-answers.pkl']) +def test_training_input_changes_rejected_by_freeze_and_direct_evaluation(tmp_path,changed_file): + manifest,_,folder=trained_fixture(tmp_path) + path=folder/changed_file + if changed_file.startswith('id2'): + mapping=pickle.loads(path.read_bytes()) + mapping[0],mapping[1]=mapping[1],mapping[0] + path.write_bytes(pickle.dumps(mapping)) + else: + path.write_bytes(path.read_bytes()+b'\n') + with pytest.raises(ValueError,match='Training input differs'): + freeze(manifest,tmp_path/'bundle',tmp_path) + with pytest.raises(ValueError,match='Training input differs'): + evaluate_manifest(manifest,input_root=tmp_path,output=tmp_path/'direct') + assert not (tmp_path/'direct').exists() + + +@pytest.mark.parametrize('missing',['training','input','profile']) +def test_required_training_bindings_cannot_be_omitted(tmp_path,missing): + manifest,record,_=trained_fixture(tmp_path) + if missing=='training': + del manifest['entries'][0]['training'] + elif missing=='input': + del record['inputs']['id2ent.pkl'] + else: + del record['profile'] + (tmp_path/'training.json').write_text(json.dumps(record)) + with pytest.raises(ValueError,match='Training provenance'): + freeze(manifest,tmp_path/'bundle',tmp_path) + with pytest.raises(ValueError,match='Training provenance'): + evaluate_manifest(manifest,input_root=tmp_path,output=tmp_path/'direct') + + +@pytest.mark.integration +@pytest.mark.skipif(not os.environ.get('DICEE_INDUCTIVE_REFERENCE_ROOT'), reason='Author dependencies and clean checkout required') +@pytest.mark.parametrize('method', ['incoming-relation', 'inductive-gnnqe', 'ultraquery-lp', 'qto']) +def test_author_training_export_parity_and_verified_execution(tmp_path, method): + name = ('WikiTopicsQuery:art' if method=='incoming-relation' else 'InductiveFB15k237Query:106' + if method=='inductive-gnnqe' else 'FB15kLogicalQuery') + manifest = fixture(tmp_path, name) + e = manifest['entries'][0] + e['method'], e['id'] = method, method+'-fixture' + if method=='incoming-relation': + (tmp_path / 'heuristic.json').write_bytes((PUBLIC / 'heuristic.json').read_bytes()) + e.update(checkpoint='heuristic.json', options={}) + elif method=='ultraquery-lp': + # The actual released LP weights, rather than a synthetic query checkpoint. + (tmp_path/'lp.pt').write_bytes((REPO / 'Experiments/query-baselines/upstream/ultra/ckpts/ultra_3g.pth').read_bytes()) + e['checkpoint']='lp.pt' + e['options']['threshold']=.8 + else: + e.update(copy.deepcopy(next(row for row in read_manifest(PUBLIC/'trained_baselines.json')['entries'] + if row['method']==method and row['dataset']==name))) + folder = tmp_path / 'data' / ('106' if method=='inductive-gnnqe' else 'FB15k-betae') + if method=='inductive-gnnqe': + queries=pickle.loads((folder/'valid_queries.pkl').read_bytes()) + (folder/'train_queries.pkl').write_bytes(pickle.dumps(queries)) + answers={s:{q:{1} for q in group} for s,group in queries.items()} + (folder/'train_answers_hard.pkl').write_bytes(pickle.dumps(answers)) + else: + (folder/'valid.txt').write_text('0\t0\t3\n') + (folder/'test.txt').write_text('1\t2\t3\n') + source = Path(os.environ['DICEE_INDUCTIVE_REFERENCE_ROOT']) if method=='inductive-gnnqe' else REPO/'Experiments/query-baselines/upstream/qto' + checkout = tmp_path/'Experiments/query-baselines/upstream'/('inductiveqe' if method=='inductive-gnnqe' else 'qto') + checkout.parent.mkdir(parents=True) + checkout.symlink_to(source, target_is_directory=True) + subprocess.run([sys.executable,'-m','benchmarks.cqa','ultraquery','train','--methods',method, + '--datasets',name,'--input-root',str(tmp_path),'--data-root','data', + '--output',str(tmp_path/'trained'),'--smoke'],check=True) + training=tmp_path/'trained'/f'{method}-{name}' + e['checkpoint']=str(training/'checkpoint.pt') + metadata=json.loads((training/'training.json').read_text()) + assert metadata['status']=='complete' and metadata['profile']=='smoke' and metadata['updates']==1 + assert metadata['checkpoint_selection']=='validation MRR only' + generated=json.loads((training/'manifest.json').read_text()) + assert generated['data_root']=='data' and generated['threads']==2 + assert generated['entries'][0]['blockers'] + assert (tmp_path/generated['entries'][0]['checkpoint']).is_file() + assert (tmp_path/generated['entries'][0]['training']).is_file() + e['training']=generated['entries'][0]['training'] + freeze(generated,tmp_path/'smoke-bundle',tmp_path) + with pytest.raises(ValueError,match='not ready for final testing'): + run_job(tmp_path/'smoke-bundle',generated['entries'][0]['id'],tmp_path,tmp_path/'blocked',phase='test') + changed=dict(metadata,checkpoint_sha256='0'*64) + (training/'training.json').write_text(json.dumps(changed)) + with pytest.raises(ValueError,match='Training provenance'): + freeze(generated,tmp_path/'invalid-training',tmp_path) + (training/'training.json').write_text(json.dumps(metadata)) + evaluate_manifest(generated,input_root=tmp_path,output=tmp_path/'direct',split='valid',limit=1) + result=json.loads((tmp_path/'direct'/generated['entries'][0]['id']/'result.json').read_text()) + assert result['inference']['paper_protocol']['training']['profile']=='smoke' + e['options']={'logic':'product'} if method=='inductive-gnnqe' else { + 'threshold':.001,'negation_scale':6,'cache_bytes':2**20,'row_batch_size':100} + context=load_benchmark(tmp_path/'data',name,split='valid').context + native=load_method(method,e['checkpoint'],context,**e['options']) + if method=='inductive-gnnqe': + assert native.provenance['checkpoint_graph_buffers']['train_graph']['training_facts_verified'] + from benchmarks.cqa.train import compatibility + compatibility() + from torchdrug.data import Graph + with torch.serialization.safe_globals([Graph]): + state=torch.load(e['checkpoint'],weights_only=True) + del state['model']['train_graph'] + torch.save(state,tmp_path/'incomplete.pt') + with pytest.raises(ValueError,match='Missing inductive checkpoint graph'): + load_method(method,tmp_path/'incomplete.pt',context,**e['options']) + freeze(manifest,tmp_path/'bundle',tmp_path) + source = (Path(os.environ['DICEE_INDUCTIVE_REFERENCE_ROOT']) if method in ('incoming-relation','inductive-gnnqe') + else REPO/'Experiments/query-baselines/upstream'/('qto' if method=='qto' else 'ultra')) + subprocess.run([sys.executable,str(EXPORTER),'--bundle',str(tmp_path/'bundle'), + '--input-root',str(tmp_path),'--entry',e['id'],'--upstream',str(source), + '--device','cpu','--output',str(tmp_path/'reference.pt')],check=True) + evidence=verify_predictions(tmp_path/'bundle',e['id'],tmp_path,tmp_path/'reference.pt',tmp_path/'evidence.json') + assert evidence['passed'] and len(evidence['queries'])==14, json.dumps(evidence) + e['verification']='evidence.json' + freeze(manifest,tmp_path/'verified',tmp_path) + if method in ('qto','inductive-gnnqe'): + # Even successful independent parity cannot promote genuine one-update + # smoke weights through the shipped manifest's empty blocker list. + assert e['blockers']==[] + with pytest.raises(ValueError,match='smoke/fixture training checkpoint'): + run_job(tmp_path/'verified',e['id'],tmp_path,tmp_path/'test',phase='test') + report=run_job(tmp_path/'verified',e['id'],tmp_path,tmp_path/'pilot',phase='pilot') + else: + report=run_job(tmp_path/'verified',e['id'],tmp_path,tmp_path/'test',phase='test') + assert report['coverage']['complete_benchmark_types'] + + +@pytest.mark.integration +@pytest.mark.skipif(not os.environ.get('DICEE_INDUCTIVE_REFERENCE_ROOT'),reason='Author runtime required') +def test_qto_training_consumes_exact_epochs(tmp_path): + fixture(tmp_path,'FB15kLogicalQuery') + folder=tmp_path/'data/FB15k-betae' + (folder/'valid.txt').write_text('0\t0\t3\n') + (folder/'test.txt').write_text('1\t2\t3\n') + checkout=tmp_path/'Experiments/query-baselines/upstream/qto' + checkout.parent.mkdir(parents=True) + checkout.symlink_to(REPO/'Experiments/query-baselines/upstream/qto',target_is_directory=True) + subprocess.run([sys.executable,'-m','benchmarks.cqa','ultraquery','train', + '--methods','qto','--datasets','FB15kLogicalQuery','--input-root',str(tmp_path), + '--data-root','data','--output',str(tmp_path/'trained'), + '--epochs','2','--batch-size','2'],check=True) + record=json.loads((tmp_path/'trained/qto-FB15kLogicalQuery/training.json').read_text()) + assert record['updates']==6 # Six triples, three batches per epoch, two epochs. + assert record['best_epoch']==1 # Retain the authors' e % valid == 0 schedule. + assert record['profile']=='custom' and record['status']=='complete' + manifest=json.loads((tmp_path/'trained/qto-FB15kLogicalQuery/manifest.json').read_text()) + entry=manifest['entries'][0] + freeze(manifest,tmp_path/'bundle',tmp_path) + subprocess.run([sys.executable,str(EXPORTER),'--bundle',str(tmp_path/'bundle'), + '--input-root',str(tmp_path),'--entry',entry['id'], + '--upstream',str(REPO/'Experiments/query-baselines/upstream/qto'), + '--device','cpu','--output',str(tmp_path/'reference.pt')],check=True) + evidence=verify_predictions(tmp_path/'bundle',entry['id'],tmp_path,tmp_path/'reference.pt',tmp_path/'evidence.json') + assert evidence['passed'] + entry['verification']='evidence.json' + freeze(manifest,tmp_path/'verified',tmp_path) + report=run_job(tmp_path/'verified',entry['id'],tmp_path,tmp_path/'test',phase='test') + assert report['protocol']['full_split'] diff --git a/benchmarks/cqa/tests/test_cqd_reference.py b/benchmarks/cqa/tests/test_cqd_reference.py new file mode 100644 index 000000000..8441dac2a --- /dev/null +++ b/benchmarks/cqa/tests/test_cqd_reference.py @@ -0,0 +1,74 @@ +"""Optional independent CQD oracle tests. + +Set DICEE_CQD_REFERENCE to the pinned upstream checkout (d1ce74164936a7c09d9147e83190da047cb39429). +No DICE package or repository-local model fixtures are needed. Tolerance here +quantifies partition rounding against the dense upstream profile; it does not +change the exported profile's score/ranking/tie verification gates. +""" + +import importlib.util +import inspect +import os +from pathlib import Path +from unittest.mock import patch + +import pytest +import torch + +QUERIES = {'1p': [0, 0], '2p': [0, 0, 2], '3p': [0, 0, 2, 1], '4p': [0, 0, 2, 1, 3], + '2i': [0, 0, 1, 2], '3i': [0, 0, 1, 2, 2, 1], '4i': [0, 0, 1, 2, 2, 1, 3, 3], + 'ip': [0, 0, 1, 2, 2], 'pi': [0, 0, 2, 1, 2], '2u': [0, 0, 1, 2, -1], 'up': [0, 0, 1, 2, -1, 2]} + + +@pytest.fixture(scope='module') +def reference(): + checkout = os.environ.get('DICEE_CQD_REFERENCE') + if not checkout: + pytest.skip('Set DICEE_CQD_REFERENCE to the optional pinned CQD checkout') + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.syspath_prepend(checkout) + from cqd import discrete + from cqd.base import CQD + path = Path(__file__).resolve().parents[1] / 'verification' / 'reference_cqd.py' + spec = importlib.util.spec_from_file_location('bounded_cqd_reference', path) + helper = importlib.util.module_from_spec(spec) + spec.loader.exec_module(helper) + yield CQD, discrete, helper + + +@pytest.mark.parametrize('hybrid', [False, True]) +@pytest.mark.parametrize('tnorm', ['prod', 'min']) +@pytest.mark.parametrize('batch_size', [2, 32]) +def test_positive_shapes_against_upstream(reference, hybrid, tnorm, batch_size): + CQD, source, helper = reference + filters = {(0, 0): [1, 2], (1, 2): [3], (2, 2): [4], (3, 0): [5], (5, 2): [6], (4, 0): [0]} + model = CQD(11, 4, 7, k=3, max_k=5, max_norm=.9 if hybrid else 1., filters=filters, + do_normalize=True, t_norm_name=tnorm).eval() + generator = torch.Generator().manual_seed(107) + with torch.no_grad(): + for parameter in model.parameters(): + parameter.copy_(torch.randn(parameter.shape, generator=generator)) + + def scoring(lhs, relation, rhs): + values = model.score_o(lhs, relation, rhs)[0] + return model.max_norm * (values - values.min()) / (values.max() - values.min()) + + for shape, flat in QUERIES.items(): + queries = torch.tensor([flat]) + function = getattr(source, 'query_' + {'2u': '2u_dnf', 'up': 'up_dnf'}.get(shape, shape)) + arguments = dict(entity_embeddings=model.embeddings[0], predicate_embeddings=model.embeddings[1], + queries=queries, filters=filters, k=model.k, max_k=model.max_k, max_norm=model.max_norm, + scoring_function=scoring, t_norm=torch.mul if tnorm == 'prod' else torch.minimum, + t_conorm=(lambda a, b: 1 - (1 - a) * (1 - b)) if tnorm == 'prod' else torch.maximum) + expected = function(**{name: arguments[name] for name in inspect.signature(function).parameters}) + with patch.object(model, 'score_o', wraps=model.score_o) as calls: + actual = helper.query_bounded(model, source, shape, queries, batch_size) + assert all(call.args[0].shape[0] <= batch_size for call in calls.call_args_list) + torch.testing.assert_close(actual, expected, atol=1e-6, rtol=1e-6, msg=(shape, hybrid, tnorm, batch_size)) + assert torch.equal(actual, helper.query_bounded(model, source, shape, queries, batch_size)) + + +def test_rejects_multiple_query_batches(reference): + _, source, helper = reference + with pytest.raises(ValueError, match='one query'): + helper.query_bounded(None, source, '1p', torch.zeros(2, 2, dtype=torch.long), 32) diff --git a/benchmarks/cqa/tests/test_download_inputs.py b/benchmarks/cqa/tests/test_download_inputs.py new file mode 100644 index 000000000..2f5c68047 --- /dev/null +++ b/benchmarks/cqa/tests/test_download_inputs.py @@ -0,0 +1,213 @@ +"""Input setup must be selective, bounded, and preserve immutable inputs.""" + +import hashlib +import io +import json +import stat +import subprocess +import sys +import zipfile + +import pytest + +from benchmarks.cqa import cli, docker, inputs +from benchmarks.cqa.manifests import REPO, SUITES, default_recipes, prepare_manifest + +DATASET = {'plus_h': 'FB15k237+H', 'ultraquery': 'WikiTopicsQuery:art'} + + +def recipes(suite, **selection): + return prepare_manifest(default_recipes(suite), suite=suite, answer_filter=SUITES[suite]['answer_filter'], **selection) + + +def test_setup_plan_downloads_only_selected_methods_and_datasets(): + jobs, weights = inputs.plan(recipes('plus_h', methods=['cqd', 'cqd-hybrid'], datasets=['NELL995+H']), 'plus_h') + assert len(jobs) == 1 + assert list(jobs[0]['folders']) == ['iscqa-compl-benchmarks/new_benchmarks/NELL995+H'] + assert weights == ['checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/nellcheckpoint'] + manifest = recipes('ultraquery', methods=['ultra-adapter'], datasets=['WikiTopicsQuery:art', 'InductiveFB15k237Query:106']) + jobs, weights = inputs.plan(manifest, 'ultraquery') + assert [job['id'] for job in jobs] == ['ultra-inductive-106', 'ultra-wikitopics'] + assert list(jobs[1]['folders']) == ['WikiTopics_QE/art'] + assert weights == ['checkpoints/ultra_3g.pth'] + with pytest.raises(ValueError): + recipes('plus_h', methods=['misspelled']) + + +def test_plus_h_plan_includes_every_released_report_category_and_corrected_filter(): + from benchmarks.cqa.difficulty import REDUCTIONS + required = inputs.plus_h_files() + for shape, labels in REDUCTIONS.items(): + for label in labels: + assert f'test-query-reduction/{shape}/{label}/test-hard-answers.pkl' in required + assert 'test-query-reduction/2in/all/test-easy-answers.pkl' in required + assert 'test-query-reduction/pni/all/test-hard-answers.pkl' in required + icews = inputs.plus_h_files(True) + assert 'KG_splits/test.txt' in icews and 'test.txt' not in icews + assert not any(name.startswith('test-query-reduction') for name in icews) + + +def test_streamed_download_rejects_bad_hash_without_touching_existing_input(tmp_path): + source = tmp_path / 'source' + source.write_bytes(b'official bytes') + target = tmp_path / 'checkpoint' + expected = hashlib.sha256(source.read_bytes()).hexdigest() + inputs.download(source.as_uri(), target, expected) + assert target.read_bytes() == b'official bytes' + source.write_bytes(b'corrupted download') + with pytest.raises(ValueError, match='SHA-256'): + inputs.download(source.as_uri(), target, expected) + assert target.read_bytes() == b'official bytes' + assert not list(tmp_path.glob('.input-*')) + with pytest.raises(ValueError, match='refusing to replace'): + inputs.install(io.BytesIO(b'different input'), target) + assert target.read_bytes() == b'official bytes' + + +def test_interrupted_stream_never_installs_partial_input_and_reads_are_bounded(tmp_path): + class Broken: + calls = 0 + + def read(self, size): + assert size == inputs.CHUNK + self.calls += 1 + if self.calls == 2: + raise OSError('connection dropped') + return b'partial' + + with pytest.raises(OSError, match='connection dropped'): + inputs.install(Broken(), tmp_path / 'checkpoint') + assert not list(tmp_path.iterdir()) + + +@pytest.mark.parametrize('unsafe', ['../escape', '/absolute', 'symlink']) +def test_archive_is_validated_before_any_file_is_installed(tmp_path, unsafe): + archive = tmp_path / 'data.zip' + with zipfile.ZipFile(archive, 'w') as zipped: + zipped.writestr('valid.txt', b'valid') + if unsafe == 'symlink': + member = zipfile.ZipInfo('symlink') + member.external_attr = (stat.S_IFLNK | 0o777) << 16 + zipped.writestr(member, '../escape') + else: + zipped.writestr(unsafe, b'unsafe') + root = tmp_path / 'inputs' + root.mkdir() + with pytest.raises(ValueError, match='Unsafe|Symlink'): + inputs.extract(archive, root, lambda name: name if name == 'valid.txt' else None) + assert not list(root.iterdir()) + + +def test_complete_public_inputs_do_not_contact_network_and_install_shipped_adapter(tmp_path, monkeypatch): + manifest = recipes('ultraquery', methods=['ultra-adapter'], datasets=['WikiTopicsQuery:art']) + jobs, weights = inputs.plan(manifest, 'ultraquery') + for job in jobs: + for name in inputs.missing_files(tmp_path, job): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'existing labels') + weight = tmp_path / weights[0] + weight.parent.mkdir(parents=True, exist_ok=True) + weight.write_bytes(b'weight fixture') + monkeypatch.setitem(inputs.WEIGHT_HASHES, weights[0], inputs.digest(weight)) + monkeypatch.setattr(inputs, 'download', lambda *a, **kw: pytest.fail('Unexpected network request')) + inputs.ensure_inputs(manifest, tmp_path, suite='ultraquery') + adapter = next(iter(manifest['entries'][0]['adapters'].values())) + assert (tmp_path / adapter).read_bytes() == (REPO / adapter).read_bytes() + inputs.ensure_inputs(manifest, tmp_path, suite='ultraquery', download_missing=False) + + +def test_custom_labels_are_not_mixed_with_public_data_and_missing_custom_weights_fail(tmp_path, monkeypatch): + manifest = recipes('plus_h', methods=['cqd'], datasets=['FB15k237+H']) + manifest['archives'] = {} + manifest['entries'][0]['checkpoint'] = 'my-model.pt' + monkeypatch.setattr(inputs, 'download', lambda *a, **kw: pytest.fail('Unexpected download for custom inputs')) + with pytest.raises(ValueError, match='custom checkpoint'): + inputs.ensure_inputs(manifest, tmp_path, suite='plus_h') + (tmp_path / 'my-model.pt').write_bytes(b'custom weights') + inputs.ensure_inputs(manifest, tmp_path, suite='plus_h') + assert not (tmp_path / manifest['data_root']).exists() + + +@pytest.mark.parametrize('suite,method', [('plus_h', 'cqd'), ('ultraquery', 'ultraquery')]) +def test_host_commands_need_only_the_standard_library(tmp_path, suite, method): + """Setup and Docker launches run with ``python -S``: no PyTorch or other site packages.""" + root = tmp_path / 'inputs' + common = [sys.executable, '-S', '-m', 'benchmarks.cqa', suite] + selection = ['--input-root', str(root), '--methods', method, '--datasets', DATASET[suite]] + output = subprocess.check_output([*common, 'setup', *selection, '--dry-run'], text=True, cwd=REPO) + assert 'https://' in output and method in output.lower() + checked = subprocess.run([*common, 'setup', *selection, '--check'], capture_output=True, text=True, cwd=REPO) + assert checked.returncode == 1 and 'Missing:' in checked.stdout + root.mkdir() + launch = subprocess.check_output([*common, 'evaluate', '--split', 'test', *selection, '--output', str(root / 'results'), + '--image', 'example:test', '--dry-run'], text=True, cwd=REPO) + assert launch.startswith('docker run') and f'example:test {suite} evaluate --input-root /inputs' in launch + assert list(root.iterdir()) == [] + + +def test_run_sets_up_the_frozen_bundle_inputs_not_the_current_recipes(tmp_path, monkeypatch): + manifest = recipes('plus_h', methods=['cqd'], datasets=['ICEWS18+H']) + study = tmp_path / 'study' + (study / 'bundle').mkdir(parents=True) + (study / 'bundle/bundle.json').write_text(json.dumps({'manifest': manifest, 'files': {'pinned': 'digest'}})) + before = (study / 'bundle/bundle.json').read_bytes() + seen = [] + + def setup(manifest, root, **kwargs): + seen.append(manifest) + raise RuntimeError('stop after setup') + + monkeypatch.setattr(inputs, 'ensure_inputs', setup) + with pytest.raises(RuntimeError, match='stop after setup'): + cli.main(['plus_h', 'run', '--phase', 'pilot', '--input-root', str(tmp_path), '--output', str(study)]) + assert seen == [manifest] + assert (study / 'bundle/bundle.json').read_bytes() == before + + +@pytest.mark.parametrize('suite', SUITES) +def test_evaluate_sets_up_selected_inputs_before_evaluation(tmp_path, monkeypatch, suite): + from benchmarks.cqa import evaluate + calls = [] + monkeypatch.setattr(inputs, 'ensure_inputs', lambda manifest, root, **kwargs: calls.append(('setup', manifest, kwargs))) + + def evaluated(manifest, **kwargs): + assert calls[-1][:2] == ('setup', manifest) + calls.append(('evaluate', manifest, kwargs)) + + monkeypatch.setattr(evaluate, 'evaluate_manifest', evaluated) + args = [suite, 'evaluate', '--split', 'test', '--methods', 'ultraquery', '--datasets', DATASET[suite], '--input-root', str(tmp_path), + '--output', str(tmp_path / 'results')] + cli.main(args) + assert [call[0] for call in calls] == ['setup', 'evaluate'] + assert [entry['dataset'] for entry in calls[0][1]['entries']] == [DATASET[suite]] + assert calls[0][2]['suite'] == suite and calls[0][2].get('download_missing', True) + calls.clear() + cli.main([*args, '--no-setup']) + assert calls[0][2]['download_missing'] is False + + +@pytest.mark.parametrize('suite', SUITES) +def test_setup_dry_run_lists_sources_without_writing(tmp_path, capsys, suite): + cli.main([suite, 'setup', '--methods', 'ultraquery', '--datasets', DATASET[suite], + '--input-root', str(tmp_path / 'absent'), '--dry-run']) + output = capsys.readouterr().out + assert 'https://' in output and 'ultraquery.pth' in output + assert not (tmp_path / 'absent').exists() + + +@pytest.mark.parametrize('suite', SUITES) +def test_docker_launch_fetches_on_host_before_isolated_container(tmp_path, monkeypatch, suite): + calls = [] + monkeypatch.setattr(inputs, 'ensure_inputs', lambda manifest, root, **kwargs: calls.append(('setup', root, kwargs['suite']))) + monkeypatch.setattr(docker, 'run', lambda image, argv, **kwargs: calls.append(('container', image, argv))) + root = tmp_path / 'inputs' + root.mkdir() + args = [suite, 'evaluate', '--split', 'test', '--input-root', str(root), '--output', str(root / 'results'), '--image', 'test', '--device', 'cpu', + '--methods', 'ultraquery', '--datasets', DATASET[suite]] + cli.main(args) + assert [call[0] for call in calls] == ['setup', 'container'] and calls[0][2] == suite + assert '--no-setup' in calls[1][2] and '--image' not in calls[1][2] + calls.clear() + cli.main([*args, '--no-setup']) + assert [call[0] for call in calls] == ['container'] diff --git a/benchmarks/cqa/tests/test_inference_difficulty.py b/benchmarks/cqa/tests/test_inference_difficulty.py new file mode 100644 index 000000000..321f7f207 --- /dev/null +++ b/benchmarks/cqa/tests/test_inference_difficulty.py @@ -0,0 +1,212 @@ +"""Answer partition, graph provenance and inference-free difficulty regressions.""" +import json +import pickle +import sqlite3 +from collections import defaultdict + +import numpy as np +import pytest + +from benchmarks.cqa.difficulty import ( + DifficultyClassifier, + difficulty_name, + export_difficulty_reports, + positive_edges, + released_labels, + root_queries, + summarize_difficulty, +) +from dicee.query_answering._checkpoint import checksum, write_json +from dicee.query_answering._query import QUERY_SHAPES, compile_query + + +def outgoing(edges): + result = defaultdict(lambda: defaultdict(set)) + for h, r, t in edges: + result[h][r].add(t) + return result + + +def test_minimum_witness_and_graph_change_with_bounded_cache(): + full = outgoing([(0, 0, 1), (0, 0, 2), (1, 1, 3), (2, 1, 3), (2, 1, 4)]) + train = outgoing([(0, 0, 1)]) + extended = outgoing([(0, 0, 1), (0, 0, 2)]) + q = (0, (0, 1)) + classifier = DifficultyClassifier(full, train, 5, cache_answers=2) + costs, maximum = classifier.classify('2p', q, {3, 4}) + assert costs == {3: 1, 4: 2} and maximum == 2 + assert classifier.cached_answers <= 2 + for relation in range(10, 20): + classifier.costs(('project', relation, ('anchor', 0))) + assert len(classifier.cache) <= 2 + assert DifficultyClassifier(full, extended, 5).classify('2p', q, {3, 4})[0] == {3: 1, 4: 1} + assert [difficulty_name(k, maximum) for k in range(3)] == ['observed', 'partial', 'full'] + with pytest.raises(ValueError, match='not true'): + classifier.classify('2p', q, {0}) + + +def test_union_chooses_branch_and_negation_constrains_intermediate_witness(): + full = outgoing([(0, 0, 2), (0, 0, 3), (1, 1, 2), (2, 2, 4), (3, 2, 4)]) + train = outgoing([(0, 0, 2), (2, 2, 4)]) + classifier = DifficultyClassifier(full, train, 5) + union = (((0, (0,)), (1, (1,)), (-1,)), (2,)) + assert classifier.classify('up', union, {4}) == ({4: 0}, 2) + # Witness 2 is cheap but violates the negated branch; witness 3 costs two. + negated = (((0, (0,)), (1, (1, -2))), (2,)) + assert classifier.classify('inp', negated, {4}) == ({4: 2}, 2) + pni = ((1, (1, 2, -2)), (0, (0,))) + assert classifier.classify('pni', pni, {2, 3}) == ({2: 0, 3: 1}, 1) + assert positive_edges(compile_query(((0, (0,)), (1, (1,)), (-1,)))) == 1 + + +def fixture_records(tmp_path): + queries = [(0, (0, 1)), (1, (0, 1))] + folder = tmp_path / 'data' + folder.mkdir() + for name, value in { + 'queries': {QUERY_SHAPES['2p']: set(queries)}, + 'easy-answers': {q: {9} for q in queries}, + 'hard-answers': {queries[0]: {3, 4}, queries[1]: {3}}, + }.items(): + (folder / f'test-{name}.pkl').write_bytes(pickle.dumps(value)) + records = root_queries(folder, ['2p']) + trace = tmp_path / 'ranks.sqlite3' + with sqlite3.connect(trace) as db: + db.execute('CREATE TABLE queries(position INTEGER, query_id TEXT, shape TEXT, answers TEXT)') + for i, q in enumerate(queries): + key = next(k for k, r in records.items() if r['query'] == q) + rows = [[3, 1, 0, 2], [4, 4, 3, 1]] if i == 0 else [[3, 2, 1, 1]] + db.execute('INSERT INTO queries VALUES (?, ?, ?, ?)', (i, key, '2p', json.dumps(rows))) + return folder, queries, records, trace + + +class ToyClassifier: + def classify(self, shape, query, hard): + return ({3: 1, 4: 2} if query[0] == 0 else {3: 1}), 2 + + +def test_mixed_answers_ties_conditional_means_and_additive_weights(tmp_path): + _, _, records, trace = fixture_records(tmp_path) + summary = summarize_difficulty(trace, records, ToyClassifier()) + rows = {(r['grouping'], r['label']): r for r in summary['rows']} + partial, full = rows['difficulty', 'partial'], rows['difficulty', 'full'] + assert partial['queries'] == 2 and full['queries'] == 1 + assert partial['sort']['mrr'] == .75 + assert partial['expected']['mrr'] == .625 # E[1/rank] for uniform ranks 1,2. + assert partial['expected']['hits1'] == .25 + assert full['sort']['mrr'] == .25 + overall = rows['overall', 'all'] + assert overall['sort']['mrr'] == .5625 + assert partial['query_weight'] == .75 and full['query_weight'] == .25 + for policy in ('sort', 'expected'): + for metric in ('mrr', 'hits1', 'hits3', 'hits10'): + assert np.isclose(sum(r['original_weight_contribution'][policy][metric] for r in (partial, full)), overall[policy][metric]) + assert rows['released_reduction', 'unavailable']['hard_answers'] == 3 + + +@pytest.mark.parametrize('mutation,match', [ + ("UPDATE queries SET query_id='wrong' WHERE position=0", 'identity'), + ("UPDATE queries SET answers='[[5,1,0,1],[4,4,3,1]]' WHERE position=0", 'exact root'), + ("UPDATE queries SET answers='[[3,1,0,1],[3,4,3,1]]' WHERE position=0", 'exact root'), + ("UPDATE queries SET answers='[[3,3,0,2],[4,4,3,1]]' WHERE position=0", 'tie block'), +]) +def test_identity_answer_and_rank_invariants(tmp_path, mutation, match): + _, _, records, trace = fixture_records(tmp_path) + with sqlite3.connect(trace) as db: + db.execute(mutation) + with pytest.raises(ValueError, match=match): + summarize_difficulty(trace, records, ToyClassifier()) + + +def test_released_partitions_missing_overlap_extra_and_provenance(tmp_path): + folder, queries, records, _ = fixture_records(tmp_path) + unavailable, info = released_labels(folder, records, dataset='FB15k237+H') + assert not unavailable and info['2p']['status'] == 'unavailable' + first = folder / 'test-query-reduction/2p/1p/test-hard-answers.pkl' + second = folder / 'test-query-reduction/2p/2p/test-hard-answers.pkl' + first.parent.mkdir(parents=True) + first.write_bytes(pickle.dumps({queries[0]: {3, 8}, queries[1]: {3}})) + with pytest.raises(FileNotFoundError): + released_labels(folder, records, dataset='FB15k237+H') + second.parent.mkdir(parents=True) + second.write_bytes(pickle.dumps({queries[0]: {4}})) + labels, info = released_labels(folder, records, dataset='FB15k237+H') + assert len(labels) == 2 and info['2p']['label_reference_graph'] == 'train+valid' + assert info['2p']['excluded_extra_answers'] == 1 + second.write_bytes(pickle.dumps({queries[0]: {3, 4}})) + with pytest.raises(ValueError, match='Overlapping'): + released_labels(folder, records, dataset='FB15k237+H') + second.write_bytes(pickle.dumps({})) + with pytest.raises(ValueError, match='Missing released'): + released_labels(folder, records, dataset='FB15k237+H') + + +def test_partial_trace_does_not_claim_complete_root_coverage(tmp_path): + _, _, records, trace = fixture_records(tmp_path) + with sqlite3.connect(trace) as db: + db.execute('DELETE FROM queries WHERE position=1') + summary = summarize_difficulty(trace, records, ToyClassifier()) + assert summary['counts']['2p']['queries'] == 1 and len(records) == 2 + + +@pytest.mark.parametrize('method', ['cone', 'qto', 'cqd-hybrid']) +@pytest.mark.parametrize('inference_graph', ['train', 'train+valid']) +def test_export_fixed_test_reference_corrected_filters_and_pending_jobs(tmp_path, method, inference_graph): + from dicee.query_answering.datasets import dataset_spec + folder, queries, records, trace = fixture_records(tmp_path) + actual = tmp_path / 'inputs' / 'data' / dataset_spec('FB15k237+H')[1] + actual.parent.mkdir(parents=True) + folder.rename(actual) + # Distinct relations to keep reciprocal facts from creating shortcuts. + (actual / 'train.txt').write_text('0 0 2\n') + (actual / 'valid.txt').write_text('1 0 2\n') + (actual / 'test.txt').write_text('2 1 3\n2 1 4\n') + (actual / 'id2ent.pkl').write_bytes(pickle.dumps({i: str(i) for i in range(10)})) + bundle, results = tmp_path / 'bundle', tmp_path / 'results' + correction = bundle / 'filters.json' + write_json(correction, {'changes': {}}) + pins = {str(p.relative_to(tmp_path / 'inputs')): checksum(p) for p in actual.glob('*')} + frozen = {'manifest': {'data_root': 'data', 'entries': [{'id': 'done'}, {'id': 'excluded'}, {'id': 'pending'}]}, + 'files': pins, 'source_sha256': 'old-code', + 'filter_corrections': {'FB15k237+H': {'path': 'filters.json', 'sha256': checksum(correction)}}} + write_json(bundle / 'bundle.json', frozen) + results.mkdir() + trace.rename(results / 'ranks.sqlite3') + overall = next(r for r in summarize_difficulty(results / 'ranks.sqlite3', records, ToyClassifier())['rows'] if r['grouping'] == 'overall') + report = {'benchmark_run': {'entry': 'done', 'bundle': checksum(bundle / 'bundle.json')}, + 'dataset': 'FB15k237+H', 'split': 'test', 'inference': {'method': method}, + 'dataset_metadata': {'inference_graph': inference_graph}, + 'protocol': {'answer_filter': 'corrected', 'full_split': True}, + 'rank_trace_sha256': checksum(results / 'ranks.sqlite3'), + 'per_shape': {'2p': overall['sort'] | {'queries': 2, 'hard_answers': 3}}, + 'additional_tie_metrics': {'expected': {'per_shape': {'2p': overall['expected']}}}} + write_json(results / 'result.json', report) + write_json(results / 'excluded' / 'result.json', report | { + 'benchmark_run': report['benchmark_run'] | {'entry': 'excluded'}}) + # An active job is represented by an unusable trace and no result; never read. + (results / 'active').mkdir() + (results / 'active' / 'ranks.sqlite3').write_text('do not open') + payload = export_difficulty_reports(results, bundle, tmp_path / 'inputs', tmp_path / 'out', entries=['done'], query_types=['2p']) + assert payload['pending_main_entries'] == ['pending'] + assert payload['hardness_reference_graph'] == 'train+valid' + assert {r['comparison_graph'] for r in payload['rows']} == {'train+valid'} + assert {r['label_reference_graph'] for r in payload['rows']} == {'train+valid'} + assert {r['inference_graph'] for r in payload['rows']} == {inference_graph} + assert {r['label'] for r in payload['rows'] if r['grouping'] == 'inferred_positive_edges'} == {'1'} + assert len(payload['details']) == 1 + assert payload['frozen_inference_source_sha256'] == 'old-code' + assert payload['postprocessor_files'] + assert all(r['answer_filter'] == 'corrected' for r in payload['rows']) + with pytest.raises(ValueError, match='new report directory'): + export_difficulty_reports(results, bundle, tmp_path / 'inputs', results / 'out') + # A direct evaluate run carries no bundle hash: refused by default, read with direct=True. + write_json(results / 'result.json', report | {'benchmark_run': {'entry': 'done', 'mode': 'direct'}}) + with pytest.raises(ValueError, match='frozen bundle'): + export_difficulty_reports(results, bundle, tmp_path / 'inputs', tmp_path / 'refused', entries=['done'], query_types=['2p']) + unbundled = export_difficulty_reports(results, bundle, tmp_path / 'inputs', tmp_path / 'direct', entries=['done'], + query_types=['2p'], direct=True) + assert unbundled['rows'] == payload['rows'] and unbundled['pending_main_entries'] == [] + assert (unbundled['result_mode'], payload['result_mode']) == ('direct', 'bundle') + with pytest.raises(ValueError, match='a direct run'): + export_difficulty_reports(results, bundle, tmp_path / 'inputs', tmp_path / 'none', entries=['excluded'], query_types=['2p'], + direct=True) diff --git a/benchmarks/cqa/tests/test_kgfm_reference.py b/benchmarks/cqa/tests/test_kgfm_reference.py new file mode 100644 index 000000000..def73a749 --- /dev/null +++ b/benchmarks/cqa/tests/test_kgfm_reference.py @@ -0,0 +1,264 @@ +"""Independent integration oracle checks, including ties and observed proofs.""" + +import importlib.util +import json +from pathlib import Path + +import pytest +import torch + +from dicee.query_answering import AdapterQuery, AdapterTrainingData, QueryAnswerer, QueryContext, QueryScoreAdapter, fit_query_adapter +from dicee.query_answering._query import QUERY_SHAPES +from dicee.query_answering.context import state_fingerprint + +spec = importlib.util.spec_from_file_location('reference_kgfm', Path(__file__).parents[1] / 'verification' / 'reference_kgfm.py') +reference_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(reference_module) + + +class Rows(torch.nn.Module): + num_entities, num_relations = 7, 4 + + def __init__(self): + super().__init__() + self.values = torch.nn.Parameter(torch.randn(7, 4, 7, generator=torch.Generator().manual_seed(8))) + + def forward_k_vs_all(self, conditions): + return self.values[conditions[:, 0], conditions[:, 1]] + + +@pytest.mark.parametrize('name', ['DistMult', 'ULTRA', 'TRIX', 'Flock']) +def test_default_adapter_fits_and_reloads_for_every_backbone(name, tmp_path): + from dicee.models import TRIX, ULTRA, DistMult, Flock + + model = {'DistMult': DistMult, 'ULTRA': ULTRA, 'TRIX': TRIX, 'Flock': Flock}[name](dict( + num_entities=7, num_relations=4, embedding_dim=8, ultra_dim=8, ultra_num_layers=2, + trix_dim=8, flock_dim=8, flock_walk_num=2, flock_walk_len=8, flock_refinements=2, flock_seed=17, + )).eval() + context = QueryContext([(0, 0, 2), (1, 1, 2)], 7, 4) + data = AdapterTrainingData('source', context, (AdapterQuery(((0, (0,)), (1, (1,))), {2, 3}),)) + before = state_fingerprint(model) + result = fit_query_adapter(model, [data], epochs=2) + assert result.adapter.feature_mode == QueryScoreAdapter().feature_mode == 'context_scores' + assert result.adapter.hidden_dim == 0 and result.adapter.weights.numel() == 16 + assert result.adapter.weights.count_nonzero() > 0 + assert state_fingerprint(model) == before + result.adapter.save(tmp_path / 'adapter.json') + restored = QueryScoreAdapter.load(tmp_path / 'adapter.json', model=model) + assert restored.feature_mode == 'context_scores' + torch.testing.assert_close(restored.weights, result.adapter.weights, atol=0, rtol=0) + + +@pytest.mark.parametrize('hidden_dim', [0, 5]) +def test_adapter_feature_alias_preserves_scores_and_saves_canonical_name(hidden_dim, tmp_path): + model = Rows() + context = QueryContext([(0, 0, 2), (1, 1, 2)], 7, 4) + expected = QueryScoreAdapter('context_scores', 1., hidden_dim=hidden_dim, + weights=torch.full((2, hidden_dim or 8), .1), + metadata={'backbone_state_sha256': state_fingerprint(model)}) + payload = dict(expected.to_dict(), feature_mode='context_scores_v1') + path = tmp_path / 'adapter.json' + path.write_text(json.dumps(payload)) + actual = QueryScoreAdapter.load(path, model=model) + raw = model.forward_k_vs_all(torch.tensor([[0, 0], [1, 1]])) + observed, base = context.features([(0, 0), (1, 1)], device='cpu') + torch.testing.assert_close(actual(raw, observed, base), expected(raw, observed, base), atol=0, rtol=0) + actual.save(tmp_path / 'canonical.json') + assert json.loads((tmp_path / 'canonical.json').read_text())['feature_mode'] == 'context_scores' + + +def instantiate(shape, counters=None): + counters = [0, 0] if counters is None else counters + if shape == 'e': + counters[0] += 1 + return (counters[0] - 1) % 7 + if shape == 'r': + counters[1] += 1 + return (counters[1] - 1) % 4 + if shape in ('n', 'u'): + return -2 if shape == 'n' else -1 + return tuple(instantiate(child, counters) for child in shape) + + +@pytest.mark.parametrize('operator', ['product', 'min']) +@pytest.mark.parametrize('cache_bytes', [0, 4096]) +@pytest.mark.parametrize('facts', ['none', 'atomic', 'both']) +def test_independent_reference_covers_all_shapes(operator, cache_bytes, facts): + model = Rows().eval() + context = QueryContext([(0, 0, 1), (0, 0, 2), (2, 1, 6), (1, 1, 3), (3, 2, 4), (4, 3, 5)], 7, 4) + adapter = QueryScoreAdapter('context_scores', float(facts != 'none'), bias_bound=8., + weights=torch.randn(2, 8, generator=torch.Generator().manual_seed(9)).double() / 5) + reference = reference_module.ReferenceKGFM(context.triples, 7, 4, adapter.to_dict(), beam_size=1, device='cpu', + restore_observed=facts == 'both') + conditions = [(h, r) for h in range(7) for r in range(4)] + with torch.no_grad(): + for start in range(0, len(conditions), 2): + batch = conditions[start:start + 2] + reference.capture(batch, model.forward_k_vs_all(torch.tensor(batch))) + engine = QueryAnswerer(model, context=context, adapter=adapter, row_batch_size=2, cache_bytes=cache_bytes, + restore_observed=facts == 'both') + queries = [instantiate(shape) for shape in QUERY_SHAPES.values()] + engine.prefetch(queries) + for query in queries: + expected = reference.predict(query, operator) + actual = engine.predict(query, beam_size=1, tnorm='prod' if operator == 'product' else 'min', return_log_scores=True) + torch.testing.assert_close(actual, expected, atol=1e-14, rtol=1e-14) + assert torch.equal(actual.argsort(descending=True), expected.argsort(descending=True)) + # Both observed first hops tie at one; ID 2 is pruned, but its proof survives. + assert (reference.predict((0, (0, 1)), operator)[6] == 0).item() == (facts == 'both') + assert (engine.predict((0, (0,)), return_log_scores=True)[1] == 0).item() == (facts != 'none') + with torch.no_grad(): + adapter.weights[1].add_(1) + assert not torch.allclose(engine.predict((0, (0,)), return_log_scores=True), reference.predict((0, (0,)))) + + +@pytest.mark.parametrize('operator', ['product', 'min']) +@pytest.mark.parametrize('facts', ['none', 'both']) +@pytest.mark.parametrize('mode', ['global', 'context_scores']) +def test_reference_applies_a_fixed_membership_threshold(operator, facts, mode): + model = Rows().eval() + context = QueryContext([(0, 0, 1), (0, 0, 2), (2, 1, 6), (1, 1, 3), (3, 2, 4), (4, 3, 5)], 7, 4) + weights = None if mode == 'global' else torch.randn(2, 8, generator=torch.Generator().manual_seed(9)).double() / 5 + conditions = [(h, r) for h in range(7) for r in range(4)] + with torch.no_grad(): + # Threshold at the median unobserved membership, so it zeroes about half of them. + memberships = QueryScoreAdapter(mode, 0., bias_bound=8., weights=weights)( + model.forward_k_vs_all(torch.tensor(conditions)), *context.features(conditions, device='cpu')).exp() + adapter = QueryScoreAdapter(mode, float(facts != 'none'), bias_bound=8., weights=weights, + membership_threshold=memberships.median().item()) + reference = reference_module.ReferenceKGFM(context.triples, 7, 4, adapter.to_dict(), beam_size=3, device='cpu', + restore_observed=facts == 'both') + with torch.no_grad(): + for start in range(0, len(conditions), 2): + batch = conditions[start:start + 2] + reference.capture(batch, model.forward_k_vs_all(torch.tensor(batch))) + # The threshold zeroes part of some rows, so ties at zero membership reach the beams. + assert any(torch.isneginf(row).any() and torch.isfinite(row).any() for row in reference.rows.values()) + engine = QueryAnswerer(model, context=context, adapter=adapter, row_batch_size=2, restore_observed=facts == 'both') + queries = [instantiate(shape) for shape in QUERY_SHAPES.values()] + engine.prefetch(queries) + for query in queries: + expected = reference.predict(query, operator) + actual = engine.predict(query, beam_size=3, tnorm='prod' if operator == 'product' else 'min', return_log_scores=True) + torch.testing.assert_close(actual, expected, atol=1e-14, rtol=1e-14) + assert torch.equal(actual.argsort(descending=True), expected.argsort(descending=True)) + unthresholded = reference_module.ReferenceKGFM(context.triples, 7, 4, dict(adapter.to_dict(), membership_threshold=0.), + beam_size=3, device='cpu') + unthresholded.capture([(0, 0)], model.forward_k_vs_all(torch.tensor([[0, 0]]))) + assert not torch.equal(unthresholded.rows[0, 0], reference.rows[0, 0]) + with pytest.raises(ValueError): + reference_module.ReferenceKGFM(context.triples, 7, 4, dict(adapter.to_dict(), membership_threshold=1.), + beam_size=3, device='cpu') + + +@pytest.mark.parametrize('operator', ['product', 'min']) +@pytest.mark.parametrize('facts', ['none', 'atomic', 'both']) +@pytest.mark.parametrize('calibration', ['minmax', 'softmax-degree', 'softmax', 'softmax-degree-ties', 'softmax-degree-ties-masked', + 'softmax-mass']) +def test_reference_applies_training_free_calibrations(operator, facts, calibration): + model = Rows().eval() + context = QueryContext([(0, 0, 1), (0, 0, 2), (2, 1, 6), (1, 1, 3), (3, 2, 4), (4, 3, 5)], 7, 4) + masked = calibration.endswith('-masked') + adapter = QueryScoreAdapter('global', float(facts != 'none'), bias_bound=8., fixed_calibration=calibration.removesuffix('-masked'), + mask_known_logits=masked) + reference = reference_module.ReferenceKGFM(context.triples, 7, 4, adapter.to_dict(), beam_size=2, device='cpu', + restore_observed=facts == 'both') + conditions = [(h, r) for h in range(7) for r in range(4)] + with torch.no_grad(): + for start in range(0, len(conditions), 2): + batch = conditions[start:start + 2] + reference.capture(batch, model.forward_k_vs_all(torch.tensor(batch))) + engine = QueryAnswerer(model, context=context, adapter=adapter, row_batch_size=2, restore_observed=facts == 'both') + queries = [instantiate(shape) for shape in QUERY_SHAPES.values()] + engine.prefetch(queries) + for query in queries: + expected = reference.predict(query, operator) + actual = engine.predict(query, beam_size=2, tnorm='prod' if operator == 'product' else 'min', return_log_scores=True) + torch.testing.assert_close(actual, expected, atol=1e-14, rtol=1e-14) + assert torch.equal(actual.argsort(descending=True), expected.argsort(descending=True)) + with pytest.raises(ValueError): + reference_module.ReferenceKGFM(context.triples, 7, 4, dict(adapter.to_dict(), weights=[[1.], [0.]]), + beam_size=2, device='cpu') + + +@pytest.mark.parametrize('operator', ['product', 'min']) +@pytest.mark.parametrize('calibration', [None, 'softmax-degree']) +def test_reference_orders_known_tails_by_the_backbone_when_asked(operator, calibration): + model = Rows().eval() + # (0, 0) has three known tails, more than the beam of two holds. + context = QueryContext([(0, 0, 1), (0, 0, 2), (0, 0, 5), (2, 1, 6), (1, 1, 3), (5, 1, 4), (3, 2, 4), (4, 3, 5)], 7, 4) + adapter = QueryScoreAdapter('global', 1., bias_bound=8., fixed_calibration=calibration) + adapter.observed_tie_break = True + reference = reference_module.ReferenceKGFM(context.triples, 7, 4, adapter.to_dict(), beam_size=2, device='cpu', + observed_ties='model') + conditions = [(h, r) for h in range(7) for r in range(4)] + with torch.no_grad(): + for start in range(0, len(conditions), 2): + batch = conditions[start:start + 2] + reference.capture(batch, model.forward_k_vs_all(torch.tensor(batch))) + engine = QueryAnswerer(model, context=context, adapter=adapter, row_batch_size=2) + plain = QueryAnswerer(model, context=context, adapter=QueryScoreAdapter('global', 1., bias_bound=8., fixed_calibration=calibration), + row_batch_size=2) + queries = [instantiate(shape) for shape in QUERY_SHAPES.values()] + engine.prefetch(queries) + plain.prefetch(queries) + changed = False + for query in queries: + expected = reference.predict(query, operator) + tnorm = 'prod' if operator == 'product' else 'min' + actual = engine.predict(query, beam_size=2, tnorm=tnorm, return_log_scores=True) + torch.testing.assert_close(actual, expected, atol=1e-14, rtol=1e-14) + assert torch.equal(actual.argsort(descending=True), expected.argsort(descending=True)) + changed |= not torch.equal(actual, plain.predict(query, beam_size=2, tnorm=tnorm, return_log_scores=True)) + assert changed + + +@pytest.mark.parametrize('operator', ['product', 'min']) +@pytest.mark.parametrize('calibration', [None, 'softmax-degree']) +def test_reference_negates_from_known_facts_when_asked(operator, calibration): + model = Rows().eval() + context = QueryContext([(0, 0, 1), (0, 0, 2), (0, 0, 5), (2, 1, 6), (1, 1, 3), (5, 1, 4), (3, 2, 4), (4, 3, 5)], 7, 4) + adapter = QueryScoreAdapter('global', 1., bias_bound=8., fixed_calibration=calibration) + reference = reference_module.ReferenceKGFM(context.triples, 7, 4, adapter.to_dict(), beam_size=2, device='cpu', negation='observed') + conditions = [(h, r) for h in range(7) for r in range(4)] + with torch.no_grad(): + for start in range(0, len(conditions), 2): + batch = conditions[start:start + 2] + reference.capture(batch, model.forward_k_vs_all(torch.tensor(batch))) + engine = QueryAnswerer(model, context=context, adapter=adapter, row_batch_size=2) + queries = [instantiate(shape) for shape in QUERY_SHAPES.values()] + engine.prefetch(queries) + tnorm = 'prod' if operator == 'product' else 'min' + changed = False + for query in queries: + expected = reference.predict(query, operator) + actual = engine.predict(query, beam_size=2, tnorm=tnorm, return_log_scores=True, negation='observed') + torch.testing.assert_close(actual, expected, atol=1e-14, rtol=1e-14) + assert torch.equal(actual.argsort(descending=True), expected.argsort(descending=True)) + changed |= not torch.equal(actual, engine.predict(query, beam_size=2, tnorm=tnorm, return_log_scores=True)) + assert changed + + +@pytest.mark.parametrize('operator', ['product', 'min']) +def test_reference_needs_no_rows_for_zero_prefix_but_requires_live_rows(operator): + adapter = QueryScoreAdapter('context_scores', 1.) + reference = reference_module.ReferenceKGFM([(0, 0, t) for t in range(7)], 7, 4, + adapter.to_dict(), beam_size=7, device='cpu') + reference.capture([(0, 0)], torch.zeros(1, 7)) + contradiction = (((0, (0,)), (0, (0, -2))), (1,)) + assert torch.isneginf(reference.predict(contradiction, operator)).all() + with pytest.raises(KeyError): + reference.predict((0, (0, 1)), operator) + + +def test_recipe_rules_of_the_opt_in_backbone_options(): + from dicee.query_answering.catalog import KGFM_ADAPTERS, KGFM_BACKBONES, check_recipe + shapes = ['1p', '2i'] + base = dict(operators={'1p': 'product', '2i': 'product'}, adapters={'product': 'a.json'}, selection_protocol='source-validation') + assert 'flock-adapter' in KGFM_ADAPTERS and KGFM_BACKBONES['flock-adapter'][0] == 'jw9730/flock' + check_recipe(dict(base, method='flock-adapter', options=dict(test_samples=1, walk_num=128)), shapes) + check_recipe(dict(base, method='ultra-adapter', options=dict(relation_conditioning='query')), shapes) + for method, options in (('ultra-adapter', dict(test_samples=1)), ('flock-adapter', dict(walk_num=0)), + ('trix-adapter', dict(relation_conditioning='query')), ('ultra-adapter', dict(relation_conditioning='inverse'))): + with pytest.raises(ValueError): + check_recipe(dict(base, method=method, options=options), shapes) diff --git a/benchmarks/cqa/tests/test_launcher.py b/benchmarks/cqa/tests/test_launcher.py new file mode 100644 index 000000000..3521ac6e9 --- /dev/null +++ b/benchmarks/cqa/tests/test_launcher.py @@ -0,0 +1,500 @@ +"""Portable preparation profiles and launcher checks without Docker or inference.""" +import json +import subprocess + +import pytest + +from benchmarks.cqa import cli +from benchmarks.cqa.manifests import prepare_manifest, read_manifest, suite_directory, validate_entry +from benchmarks.cqa.study import entry_inputs_identity, freeze, read, run_job, verify_bundle +from benchmarks.cqa.tests.test_protocol import paper_fixture + +PUBLIC = suite_directory('plus_h') +IMAGE = ['--image', 'example/plus-h:local', '--dry-run'] + + +@pytest.mark.parametrize('suite', ['plus_h', 'ultraquery']) +def test_h100_profile_preserves_protocol_and_records_execution_options(suite): + paths = [suite_directory(suite) / name for name in ('baselines.json', 'kgfm_adapters.json')] + settings = dict(suite=suite, answer_filter='corrected' if suite == 'plus_h' else 'released') + baseline = prepare_manifest(paths, **settings) + h100 = prepare_manifest(paths, hardware_profile='h100', **settings) + for original, changed in zip(baseline['entries'], h100['entries']): + validate_entry(changed) + for key in original.keys() - {'options', 'reference'}: + assert changed[key] == original[key] + options = changed['options'] + if original['method'].endswith('-adapter'): + assert options['raw_cache_device'] == 'cpu' and options['raw_cache_granularity'] == 'row' + assert options['row_batch_size'] == options['backend_batch_size'] == 16 + assert options['cache_bytes'] == 512 * 2**20 and options['raw_cache_bytes'] == 24 * 2**30 + assert options['beam_size'] == original['options']['beam_size'] + else: + assert changed == original + override = prepare_manifest(paths, hardware_profile='h100', kgfm_batch_size=8, **settings) + assert all(e['options']['backend_batch_size'] == e['options']['row_batch_size'] == 8 + for e in override['entries'] if e['method'].endswith('-adapter')) + assert prepare_manifest(paths, **settings) == baseline + + +@pytest.mark.parametrize('settings', [dict(hardware_profile='bad'), dict(kgfm_batch_size=0), + dict(kgfm_batch_size=True), dict(kgfm_batch_size=1, methods=['qto'])]) +def test_invalid_hardware_settings_fail_before_inputs(settings): + with pytest.raises(ValueError): + prepare_manifest([PUBLIC / 'baselines.json'], **settings) + + +@pytest.mark.parametrize('suite', ['plus_h', 'ultraquery']) +def test_hardware_profile_cli_and_container_forwarding(tmp_path, capsys, suite): + output = tmp_path / 'run' + cli.main([suite, 'evaluate', '--split', 'test', '--hardware-profile', 'h100', '--kgfm-batch-size', '16', '--methods', 'ultra-adapter', + '--output', str(output), '--dry-run']) + entries = json.loads(capsys.readouterr().out)['entries'] + assert all(e['options']['backend_batch_size'] == 16 and e['options']['raw_cache_granularity'] == 'row' for e in entries) + common = ['--input-root', str(tmp_path), '--output', str(output), *IMAGE] + flags = ['--hardware-profile', 'h100', '--kgfm-batch-size', '16'] + cli.main([suite, 'prepare', *common, *flags]) + assert '--hardware-profile h100 --kgfm-batch-size 16' in capsys.readouterr().out + assert not output.exists() + # Frozen bundles fix the execution options; a run cannot change them. + with pytest.raises(SystemExit): + cli.main([suite, 'run', *common, *flags]) + assert 'unrecognized arguments' in capsys.readouterr().err + + +def test_profiles_preserve_beams_and_per_shape_settings(): + paths = [PUBLIC / name for name in ('baselines.json', 'kgfm_adapters.json')] + reference = prepare_manifest(paths, profile='reference') + bounded = prepare_manifest(paths, profile='bounded') + original = sum((read_manifest(path)['entries'] for path in paths), []) + assert reference['entries'] == original + assert reference['answer_filter'] == bounded['answer_filter'] == 'corrected' + assert prepare_manifest(paths, answer_filter='released')['answer_filter'] == 'released' + assert len(bounded['entries']) == len(reference['entries']) + for expected, actual in zip(reference['entries'], bounded['entries']): + if expected['method'] not in ('cqd', 'cqd-hybrid'): + assert actual == expected + continue + assert actual['options']['reference_batching'] is False + assert actual['options']['row_batch_size'] == actual['options']['final_batch_size'] == 32 + for key, value in expected['options'].items(): + if key not in ('reference_batching', 'row_batch_size', 'final_batch_size'): + assert actual['options'][key] == value + assert actual['query_types'] == expected['query_types'] + assert actual['query_batch_size'] == expected['query_batch_size'] + assert 'bounded' in actual['reference']['status'] + selected = prepare_manifest(paths, entries=[original[0]['id']]) + assert [entry['id'] for entry in selected['entries']] == [original[0]['id']] + with pytest.raises(ValueError, match='distinct'): + prepare_manifest(paths, entries=['missing']) + + +def test_method_dataset_query_combinations_preserve_recipes_and_trim_overrides(): + paths = [PUBLIC / name for name in ('baselines.json', 'kgfm_adapters.json')] + original = prepare_manifest(paths, profile='reference') + selected = prepare_manifest(paths, profile='reference', methods=['cqd', 'qto', 'ultra-adapter'], + datasets=['FB15k237+H', 'NELL995+H'], query_types=['epfo', '2in'], atomic_negation=True) + assert len(selected['entries']) == 6 + assert {(e['method'], e['dataset']) for e in selected['entries']} == { + (method, dataset) for method in ('cqd', 'qto', 'ultra-adapter') for dataset in ('FB15k237+H', 'NELL995+H')} + for entry in selected['entries']: + validate_entry(entry) + assert len(entry['query_types']) == 12 + assert '2in' in entry['query_types'] and 'pni' not in entry['query_types'] + assert set(entry['options'].get('per_shape', {})) <= set(entry['query_types']) + if 'operators' in entry: + assert set(entry['operators']) == set(entry['query_types']) + baseline = next(e for e in original['entries'] if e['id'] == entry['id']) + assert entry['checkpoint'] == baseline['checkpoint'] + assert {k: v for k, v in entry['options'].items() if k not in ('atomic_negation', 'per_shape')} == { + k: v for k, v in baseline['options'].items() if k not in ('atomic_negation', 'per_shape')} + assert prepare_manifest(paths, profile='reference') == original + assert prepare_manifest(paths, methods=['all'], datasets=['all']) == prepare_manifest(paths) + + +@pytest.mark.parametrize('selection', [dict(methods=['missing']), dict(datasets=['missing']), + dict(methods=['cqd', 'cqd']), dict(query_types=['bad']), + dict(query_types=['all', '2p']), dict(query_types=['2p', '2p']), + dict(methods=['qto'], entries=['cqd-FB15k237+H'])]) +def test_invalid_combinations_fail_before_opening_inputs(selection): + with pytest.raises(ValueError): + prepare_manifest([PUBLIC / 'baselines.json'], **selection) + + +@pytest.mark.parametrize('query_group', [None, 'all', 'epfo', 'negation']) +def test_evaluate_dry_run_resolves_default_cqd_coverage_without_inputs(tmp_path, capsys, query_group): + from dicee.query_answering._query import PLUS_H_SHAPES + output = tmp_path / 'no-results' + args = ['plus_h', 'evaluate', '--split', 'test', '--methods', 'cqd', 'cqd-hybrid', '--input-root', str(tmp_path / 'missing-inputs'), + '--output', str(output), '--dry-run'] + if query_group is not None: + args += ['--query-types', query_group] + cli.main(args) + manifest = json.loads(capsys.readouterr().out) + assert {(entry['method'], entry['dataset']) for entry in manifest['entries']} == { + (method, dataset) for method in ('cqd', 'cqd-hybrid') + for dataset in ('FB15k237+H', 'ICEWS18+H', 'NELL995+H')} + expected = set(PLUS_H_SHAPES) + if query_group in ('epfo', 'negation'): + expected = {shape for shape in expected if ('n' in shape) == (query_group == 'negation')} + for entry in manifest['entries']: + validate_entry(entry) + assert set(entry['query_types']) == expected + assert entry['options']['atomic_negation'] + assert not entry['options']['reference_batching'] + assert set(entry['options']['per_shape']) <= expected + assert any('signed-atom negation with +H scoring' in note for note in entry['reference']['notes']) + assert not output.exists() + + +def test_direct_subset_evaluation_scores_only_selected_methods_datasets_and_queries(tmp_path, monkeypatch): + import torch + + from dicee.models.complex import ComplEx + from dicee.query_answering.methods.cqd import CQD + + manifest = paper_fixture(tmp_path) + model = ComplEx(dict(model='ComplEx', num_entities=8, num_relations=4, embedding_dim=8, normalization=None)) + torch.save(model.state_dict(), tmp_path / 'complex.pt') + for method in ('cqd', 'cqd-hybrid'): + manifest['entries'].append(dict(manifest['entries'][0], id=f'{method}-FB15k237+H', method=method, + checkpoint='complex.pt', options={'beam_size': 2, 'atomic_negation': True, + 'per_shape': {'2p': {'beam_size': 3}}})) + manifest['entries'].append(dict(manifest['entries'][-1], id='excluded-dataset', dataset='NELL995+H', checkpoint='missing.pt')) + path = tmp_path / 'manifest.json' + path.write_text(json.dumps(manifest)) + output = tmp_path / 'results' + args = ['plus_h', 'evaluate', '--split', 'test', '--manifests', str(path), '--answer-filter', 'released', '--methods', 'cqd', 'cqd-hybrid', + '--datasets', 'FB15k237+H', '--query-types', 'negation', '--input-root', str(tmp_path), '--output', str(output), + '--device', 'cpu', '--threads', '2'] + cli.main(args) + assert sorted(p.name for p in output.iterdir()) == ['cqd-FB15k237+H', 'cqd-hybrid-FB15k237+H'] + for method in ('cqd', 'cqd-hybrid'): + report_path = output / f'{method}-FB15k237+H' / 'result.json' + report = json.loads(report_path.read_text()) + assert report['queries'] == 5 + assert set(report['per_shape']) == {'2in', '3in', 'inp', 'pin', 'pni'} + assert report['inference']['configuration']['atomic_negation'] + assert report['inference']['negation_scope'] == 'CQD-A signed-atom negation with +H scoring' + monkeypatch.setattr(CQD, 'predict', lambda *args: pytest.fail('Completed subset was recomputed')) + cli.main(args) + + +def test_observed_fact_recipes_reuse_checkpoints_and_select_only_kgfms(): + paths = [PUBLIC / name for name in ('baselines.json', 'kgfm_adapters.json')] + originals = {entry['id']: entry for entry in prepare_manifest(paths)['entries']} + expanded = prepare_manifest(paths, observed_facts=['all'])['entries'] + assert len(expanded) == 18 + for entry in expanded: + validate_entry(entry) + mode = entry['options']['observed_facts'] + original = originals[entry['id'].removesuffix(f'-facts-{mode}')] + assert entry['method'].endswith('-adapter') + assert {k: v for k, v in entry.items() if k not in ('id', 'options', 'reference')} == { + k: v for k, v in original.items() if k not in ('id', 'options', 'reference')} + assert {k: v for k, v in entry['options'].items() if k != 'observed_facts'} == original['options'] + selected = prepare_manifest(paths, observed_facts=['none', 'atomic'])['entries'] + assert len(selected) == 12 + assert {entry['options']['observed_facts'] for entry in selected} == {'none', 'atomic'} + one = prepare_manifest(paths, observed_facts=['all'], entries=[expanded[0]['id']])['entries'] + assert one == expanded[:1] + for invalid in ([], ['all', 'none'], ['atomic', 'atomic'], ['invalid']): + with pytest.raises(ValueError, match='observed-fact'): + prepare_manifest(paths, observed_facts=invalid) + one[0]['options']['observed_facts'] = 'invalid' + with pytest.raises(ValueError, match='Observed facts'): + validate_entry(one[0]) + + +def test_graph_ablation_freezes_weights_and_reuses_graph_independent_methods(): + paths = [PUBLIC / name for name in ('baselines.json', 'kgfm_adapters.json')] + original = {e['id']: e for e in prepare_manifest(paths)['entries']} + expanded = prepare_manifest(paths, inference_graphs=['train', 'train+valid'])['entries'] + assert len(expanded) == 45 + assert sum(1 + int(e.get('adapter_ablation', False)) for e in expanded) == 57 + for entry in expanded: + validate_entry(entry) + if entry['method'] in ('cone', 'clmpt', 'cqd'): + assert entry['id'] in original and entry['reference']['graph_independent'] + else: + base = original[entry['graph_ablation']] + assert entry['options'] == base['options'] + assert entry['checkpoint'] == base['checkpoint'] + assert entry.get('adapters') == base.get('adapters') + assert entry.get('operators') == base.get('operators') + for invalid in ([], ['train', 'train'], ['test']): + with pytest.raises(ValueError, match='inference graphs'): + prepare_manifest(paths, inference_graphs=invalid) + + +def test_public_test_graph_defaults_and_explicit_train_override(): + from dicee.query_answering.method_evaluation import GRAPH_INDEPENDENT_METHODS + paths = [PUBLIC / name for name in ('baselines.json', 'kgfm_adapters.json')] + defaults = prepare_manifest(paths)['entries'] + for entry in defaults: + expected = 'train' if entry['method'] in GRAPH_INDEPENDENT_METHODS else 'train+valid' + assert entry['inference_graph'] == expected + if entry['method'] in ('gnnqe', 'ultraquery', 'qto'): + assert entry['reference']['inference_graph'] == 'train' + overridden = prepare_manifest(paths, inference_graphs=['train'])['entries'] + assert len(overridden) == len(defaults) + assert all(entry['inference_graph'] == 'train' for entry in overridden) + + +def test_container_dry_run_forwards_selection_and_rejects_bad_arguments(tmp_path, capsys): + output = tmp_path / 'run' + common = ['--input-root', str(tmp_path), '--output', str(output), *IMAGE] + + def launched(command, *extra): + cli.main(['plus_h', command, *common, *extra]) + return capsys.readouterr().out + + result = launched('prepare', '--entries', 'cqd-FB15k237+H') + assert result.startswith('docker run --rm --network none --read-only') + assert f'source={tmp_path},target=/inputs,readonly' in result and f'source={output},target=/results' in result + assert 'example/plus-h:local plus_h prepare --input-root /inputs --output /results --entries cqd-FB15k237+H --no-setup' in result + assert not output.exists() + assert '--observed-facts none atomic' in launched('prepare', '--observed-facts', 'none', '--observed-facts', 'atomic') + assert '--inference-graphs train train+valid' in launched('prepare', '--inference-graphs', 'train', '--inference-graphs', 'train+valid') + assert '--answer-filter released' in launched('prepare', '--answer-filter', 'released') + for extra in (['--profile', 'invalid'], ['--output', str(tmp_path.parent / 'outside')], ['--unknown', 'x']): + with pytest.raises(SystemExit): + launched('prepare', *extra) + assert 'plus_h run --input-root /inputs --output /results --no-setup' in launched('run') + selected = launched('run', '--entries', 'ultra-product-14type-FB15k237+H-facts-none', '--entries', 'trix-product-14type-NELL995+H-facts-atomic') + assert '--entries ultra-product-14type-FB15k237+H-facts-none trix-product-14type-NELL995+H-facts-atomic' in selected + for extra in (['--answer-filter', 'corrected'], ['--observed-facts', 'none']): + with pytest.raises(SystemExit): + launched('run', *extra) + + +def test_container_forwards_combination_selectors_and_keeps_queries_frozen(tmp_path, capsys): + common = ['--input-root', str(tmp_path), '--output', str(tmp_path / 'run'), *IMAGE] + + def launched(command, *extra): + cli.main(['plus_h', command, *common, *extra]) + return capsys.readouterr().out + + result = launched('evaluate', '--methods', 'cqd', 'cqd-hybrid', '--datasets', 'FB15k237+H', 'NELL995+H', '--query-types', 'negation', + '--atomic-negation', '--split', 'valid', '--max-queries-per-shape', '3') + assert '--methods cqd cqd-hybrid --datasets FB15k237+H NELL995+H --query-types negation --atomic-negation' in result + assert '--split valid --max-queries-per-shape 3' in result + assert not (tmp_path / 'run').exists() + assert '--methods qto --datasets ICEWS18+H --query-types 2p 3p' in launched('prepare', '--methods', 'qto', '--datasets', 'ICEWS18+H', + '--query-types', '2p', '3p') + assert '--methods qto --datasets ICEWS18+H' in launched('run', '--methods', 'qto', '--datasets', 'ICEWS18+H') + for command, extra in [('run', ['--query-types', '2p']), ('run', ['--atomic-negation']), ('run', ['--split', 'valid']), + ('evaluate', ['--split', 'valid', '--methods']), ('evaluate', ['--split', 'valid', '--max-queries-per-shape', '0']), + ('evaluate', ['--methods', 'qto'])]: + with pytest.raises(SystemExit): + launched(command, *extra) + + +class _FakeWorkers: + """Popen stand-in: each worker stays running for one poll, then exits.""" + + def __init__(self, failing=()): + self.started, self.active, self.peak, self.failing = [], {}, {}, set(failing) + outer = self + + class Worker: + pid = 7 + + def __init__(self, command, *, env, **kwargs): + self.entry, self.gpu, self.returncode, self.polls = command[0], env['CUDA_VISIBLE_DEVICES'], None, 0 + outer.started.append((self.entry, self.gpu)) + outer.active[self.gpu] = outer.active.get(self.gpu, 0) + 1 + outer.peak[self.gpu] = max(outer.peak.get(self.gpu, 0), outer.active[self.gpu]) + + def poll(self): + self.polls += 1 + if self.polls < 2: + return None + if self.returncode is None: + outer.active[self.gpu] -= 1 + self.returncode = int(self.entry in outer.failing) + return self.returncode + + self.module = __import__('types').SimpleNamespace(Popen=Worker, DEVNULL=subprocess.DEVNULL, STDOUT=subprocess.STDOUT) + + +def test_parallel_workers_order_long_entries_first_and_bound_each_gpu(tmp_path, monkeypatch): + fake = _FakeWorkers() + monkeypatch.setattr(cli, 'subprocess', fake.module) + monkeypatch.setattr(cli.time, 'sleep', lambda seconds: None) + methods = ['cone', 'ultraquery', 'trix-adapter', 'cqd', 'ultra-adapter', 'gnnqe', 'clmpt'] + jobs = [(f'e{i}-{method}', method, [f'e{i}-{method}'], tmp_path / f'{i}.log') for i, method in enumerate(methods)] + assert cli.run_workers(jobs, gpus=['0', 'GPU-a'], workers_per_gpu=2, status=tmp_path / 'status.json') == [] + order = [entry.split('-', 1)[1] for entry, _ in fake.started] + assert order[:4] == ['trix-adapter', 'ultra-adapter', 'gnnqe', 'cqd'] and sorted(order) == sorted(methods) + assert {gpu for _, gpu in fake.started} == {'0', 'GPU-a'} and max(fake.peak.values()) == 2 + assert json.loads((tmp_path / 'status.json').read_text())['running'] == {} + + +def test_parallel_failure_stops_new_entries_but_lets_running_workers_finish(tmp_path, monkeypatch): + fake = _FakeWorkers(failing={'a'}) + monkeypatch.setattr(cli, 'subprocess', fake.module) + monkeypatch.setattr(cli.time, 'sleep', lambda seconds: None) + jobs = [(name, 'cone', [name], tmp_path / f'{name}.log') for name in ('a', 'b', 'c')] + assert cli.run_workers(jobs, gpus=['0', '1'], workers_per_gpu=1, status=tmp_path / 'status.json') == [('a', 1)] + assert [entry for entry, _ in fake.started] == ['a', 'b'] + assert json.loads((tmp_path / 'status.json').read_text())['failed'] == ['a'] + + +def pilot_study(tmp_path): + """A frozen two-entry study and the fake worker processes that run its pilot in this process.""" + from types import SimpleNamespace + manifest = paper_fixture(tmp_path) + manifest['entries'].append(dict(manifest['entries'][0], id='second')) + study = tmp_path / 'study' + freeze(manifest, study / 'bundle', tmp_path) + seen = [] + + class Worker: + pid, returncode = 1, 0 + + def __init__(self, command, *, env, **kwargs): + entry = command[command.index('--entries') + 1] + device = command[command.index('--device') + 1] if '--device' in command else 'cuda' + seen.append((entry, env.get('CUDA_VISIBLE_DEVICES'), device)) + run_job(study / 'bundle', entry, tmp_path, study / 'pilot' / entry) + + def poll(self): + return 0 + + workers = SimpleNamespace(Popen=Worker, DEVNULL=subprocess.DEVNULL, STDOUT=subprocess.STDOUT) + return study, ['plus_h', 'run', '--phase', 'pilot', '--input-root', str(tmp_path), '--output', str(study)], workers, seen + + +def test_parallel_run_uses_fresh_workers_and_resumes_like_sequential_runs(tmp_path, monkeypatch): + study, args, workers, seen = pilot_study(tmp_path) + monkeypatch.setattr(cli, 'subprocess', workers) + args += ['--device', 'cuda'] + cli.main(args + ['--gpus', '3', '5']) + assert sorted(seen) == [('cone-fixture', '3', 'cuda'), ('second', '5', 'cuda')] + assert read(study / 'pilot/status.json') == dict(state='complete', entries=['cone-fixture', 'second']) + stamp = (study / 'pilot/second/result.json').stat().st_mtime_ns + cli.main(args + ['--gpus', '0', '--workers-per-gpu', '2']) + assert (study / 'pilot/second/result.json').stat().st_mtime_ns == stamp + for invalid in (['--gpus', '0', '0'], ['--gpus', '0', '--workers-per-gpu', '0'], ['--workers-per-gpu', '2']): + with pytest.raises(SystemExit): + cli.main(args + invalid) + with pytest.raises(SystemExit): + cli.main(args[:-2] + ['--device', 'cpu', '--gpus', '0']) + + +def test_container_forwards_parallel_gpus_only_for_evaluate_verify_and_run(tmp_path, capsys): + common = ['--input-root', str(tmp_path), '--output', str(tmp_path / 'out'), *IMAGE] + cli.main(['plus_h', 'run', *common, '--gpus', '0', '1', 'GPU-2c', '--workers-per-gpu', '2']) + assert '--gpus 0 1 GPU-2c --workers-per-gpu 2' in capsys.readouterr().out + cli.main(['plus_h', 'evaluate', *common, '--split', 'test', '--gpus', '1']) + assert '--gpus 1' in capsys.readouterr().out + cli.main(['plus_h', 'verify', *common, '--gpus', '0', '1']) + assert '--gpus 0 1' in capsys.readouterr().out + for command, extra in [('prepare', ['--gpus', '0']), ('run', ['--workers-per-gpu', '2']), ('run', ['--gpus'])]: + with pytest.raises(SystemExit): + cli.main(['plus_h', command, *common, *extra]) + + +def test_run_subsets_share_lock_and_resume_without_recomputing(tmp_path, monkeypatch, capsys): + from dicee.query_answering._checkpoint import BenchmarkCheckpoint, checksum + study, args, workers, seen = pilot_study(tmp_path) + monkeypatch.setattr(cli, 'subprocess', workers) + args += ['--device', 'cpu'] + cli.main(args + ['--entries', 'cone-fixture']) + first_result = study / 'pilot/cone-fixture/result.json' + stamp = first_result.stat().st_mtime_ns + assert not (study / 'pilot/second/result.json').exists() + cli.main(args + ['--entries', 'second']) + cli.main(args) + assert first_result.stat().st_mtime_ns == stamp + assert read(study / 'pilot/status.json')['entries'] == ['cone-fixture', 'second'] + identity = dict(bundle=checksum(study / 'bundle/bundle.json'), phase='pilot', device='cpu', pilot_queries=2) + with BenchmarkCheckpoint(study / 'pilot/suite', identity): + with pytest.raises(RuntimeError, match='already running'): + cli.main(args + ['--entries', 'second']) + assert [entry for entry, _, _ in seen] == ['cone-fixture', 'second', 'cone-fixture', 'second'] + with pytest.raises(SystemExit): + cli.main(args + ['--pilot-queries', '3']) + assert 'settings changed' in capsys.readouterr().err + + +def test_verification_runs_each_entry_in_a_fresh_process_and_requires_oracles(tmp_path, monkeypatch, capsys): + study = tmp_path / 'study' + manifest = paper_fixture(tmp_path) + manifest['entries'].append(dict(manifest['entries'][0], id='cqd-example', method='cqd', options={'atomic_negation': True})) + freeze(manifest, study / 'bundle', tmp_path) + references, comparisons = tmp_path / 'references', tmp_path / 'dense' + references.mkdir() + comparisons.mkdir() + args = ['plus_h', 'verify', '--input-root', str(tmp_path), '--output', str(study), '--references', str(references), + '--comparison-references', str(comparisons)] + + def rejected(message): + with pytest.raises(SystemExit): + cli.main(args) + assert message in capsys.readouterr().err + + rejected('Missing baseline oracle cone-fixture.pt') + for name in ('cone-fixture', 'cqd-example'): + (references / f'{name}.pt').touch() + rejected('Missing comparison oracle') + (comparisons / 'cqd-example.pt').touch() + commands = [] + + def parity(command): + commands.append(command) + evidence = command[command.index('--output') + 1] + cli.write_json(evidence, dict(passed=True)) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(cli.subprocess, 'run', parity) + cli.main(args) + assert [command[command.index('--entry') + 1] for command in commands] == ['cone-fixture', 'cqd-example'] + assert all(command[5] == 'parity' for command in commands) + assert '--comparison-reference' in commands[1] and '--comparison-reference' not in commands[0] + verified = read(study / 'verified-manifest.json') + assert [entry['verification'] for entry in verified['entries']] == ['study/evidence/cone-fixture.json', 'study/evidence/cqd-example.json'] + assert (study / 'verified-bundle/bundle.json').is_file() + + +def test_verification_reuses_passing_parity_evidence_for_the_same_oracle(tmp_path, monkeypatch): + from dicee.query_answering._checkpoint import checksum + from dicee.query_answering.context import fingerprint + study = tmp_path / 'study' + freeze(paper_fixture(tmp_path), study / 'bundle', tmp_path) + bundle = verify_bundle(study / 'bundle', tmp_path) + entry = bundle['manifest']['entries'][0] + references = tmp_path / 'references' + references.mkdir() + (references / f'{entry["id"]}.pt').write_bytes(b'oracle') + evidence = dict(passed=True, source_sha256=bundle['source_sha256'], inputs_sha256=entry_inputs_identity(bundle, entry), + entry_sha256=fingerprint({k: v for k, v in entry.items() if k != 'verification'}), + reference_sha256=checksum(references / f'{entry["id"]}.pt')) + cli.write_json(study / 'evidence' / f'{entry["id"]}.json', dict(evidence, reference_sha256='another oracle')) + commands = [] + + def parity(command): + commands.append(command) + cli.write_json(command[command.index('--output') + 1], evidence) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(cli.subprocess, 'run', parity) + args = ['plus_h', 'verify', '--input-root', str(tmp_path), '--output', str(study), '--references', str(references)] + cli.main(args) + assert len(commands) == 1 + # Evidence for the same entry, inputs, source and oracle is reused. + (study / 'verified-bundle').rename(tmp_path / 'first-verified-bundle') + cli.main(args) + assert len(commands) == 1 and (study / 'verified-bundle/bundle.json').is_file() + + +def test_container_rejects_retagged_image_before_launch(tmp_path, monkeypatch): + from benchmarks.cqa import docker + (tmp_path / 'container-image.json').write_text(json.dumps({'image_id': 'sha256:original'})) + monkeypatch.setattr(docker.subprocess, 'check_output', lambda *a, **k: json.dumps([{'Id': 'sha256:changed'}])) + launches = [] + monkeypatch.setattr(docker.subprocess, 'run', lambda *a, **k: launches.append(a)) + with pytest.raises(ValueError, match='Image differs'): + docker.run('example/plus-h:local', ['plus_h', 'verify'], inputs=tmp_path, results=tmp_path) + assert launches == [] diff --git a/benchmarks/cqa/tests/test_manifests.py b/benchmarks/cqa/tests/test_manifests.py new file mode 100644 index 000000000..be30c2654 --- /dev/null +++ b/benchmarks/cqa/tests/test_manifests.py @@ -0,0 +1,36 @@ +"""Compact public recipes resolve to exactly the pinned benchmark entries.""" + +import json +from pathlib import Path + +import pytest + +from benchmarks.cqa.manifests import REPO, expand, merge, read_manifest +from dicee.query_answering.context import fingerprint + +PINNED = json.loads(Path(__file__).with_name('recipe_fingerprints.json').read_text()) + + +@pytest.mark.parametrize('name', sorted(PINNED)) +def test_public_recipes_expand_to_the_pinned_entries(name): + """Any change to a public recipe must update recipe_fingerprints.json in the same review.""" + entries = read_manifest(REPO / 'benchmarks' / name)['entries'] + assert {entry['id']: fingerprint(entry) for entry in entries} == PINNED[name] + assert len(entries) == len(PINNED[name]) + + +def test_entries_merge_defaults_recursively_and_fill_dataset_ids(): + recipes = [{'defaults': {'id': 'm-{dataset}', 'method': 'm', 'options': {'a': 1, 'b': {'c': 2}}, 'query_types': ['1p', '2p']}, + 'entries': [{'dataset': 'x'}, {'dataset': 'y', 'id': 'custom', 'options': {'b': {'d': 3}}, 'query_types': ['1p']}]}] + assert expand(recipes) == [ + {'id': 'm-x', 'method': 'm', 'options': {'a': 1, 'b': {'c': 2}}, 'query_types': ['1p', '2p'], 'dataset': 'x'}, + {'id': 'custom', 'method': 'm', 'options': {'a': 1, 'b': {'c': 2, 'd': 3}}, 'query_types': ['1p'], 'dataset': 'y'}] + defaults = {'options': {'a': 1}} + assert merge(defaults, {'options': None}) == {'options': None} and defaults == {'options': {'a': 1}} + + +def test_resolved_and_bundled_manifests_read_unchanged(tmp_path): + manifest = dict(version=1, data_root='data', entries=[dict(id='a', method='cone', dataset='FB15k237+H')]) + (tmp_path / 'manifest.json').write_text(json.dumps(manifest)) + (tmp_path / 'bundle.json').write_text(json.dumps(dict(manifest=manifest, files={}))) + assert read_manifest(tmp_path / 'manifest.json') == read_manifest(tmp_path / 'bundle.json') == manifest diff --git a/benchmarks/cqa/tests/test_protocol.py b/benchmarks/cqa/tests/test_protocol.py new file mode 100644 index 000000000..72b2048ff --- /dev/null +++ b/benchmarks/cqa/tests/test_protocol.py @@ -0,0 +1,729 @@ +"""Paper protocol integrity, batch replay, and inference-free tie analysis.""" + +import itertools +import os +import pickle +import shutil +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest +import torch + +from benchmarks.cqa.oracles import verify_predictions +from benchmarks.cqa.reports import export_reports, summarize_trace, trace_queries +from benchmarks.cqa.study import SOURCE_ROOT, code_identity, freeze, read, run_job, verify_bundle +from dicee.query_answering import BenchmarkQuery, QueryBenchmark, QueryContext +from dicee.query_answering._checkpoint import BenchmarkCheckpoint, write_json +from dicee.query_answering._query import PLUS_H_SHAPES, QUERY_SHAPES +from dicee.query_answering.benchmark import QueryMetrics, evaluate_benchmark +from dicee.query_answering.context import state_fingerprint +from dicee.query_answering.datasets import dataset_spec +from dicee.query_answering.methods import ConE +from dicee.query_answering.score_adapter import QueryScoreAdapter + + +def plus_h_fixture(root, name): + folder = root / dataset_spec(name)[1] + folder.mkdir(parents=True) + graph = folder / 'KG_splits' if name == 'ICEWS18+H' else folder + graph.mkdir(exist_ok=True) + for split, triples in [('train', [(0, 0, 1), (1, 2, 2)]), ('valid', [(0, 0, 3), (3, 2, 4)]), + ('test', [(0, 0, 5), (5, 2, 6)])]: + inverse = [(t, r + 1, h) for h, r, t in triples] + (graph / f'{split}.txt').write_text(''.join(f'{h}\t{r}\t{t}\n' for h, r, t in triples + inverse)) + def dump(name, value): + (folder / name).write_bytes(pickle.dumps(value)) + dump('id2ent.pkl', {i: str(i) for i in range(8)}) + dump('id2rel.pkl', {i: str(i) for i in range(4)}) + def instantiate(shape, counts): + if shape in ('n', 'u'): + return -2 if shape == 'n' else -1 + if shape in ('e', 'r'): + index = 0 if shape == 'e' else 1 + value = counts[index] + counts[index] += 1 + return value % (8 if index == 0 else 4) + return tuple(instantiate(part, counts) for part in shape) + queries = {QUERY_SHAPES[name]: {instantiate(QUERY_SHAPES[name], [0, 0])} for name in PLUS_H_SHAPES} + for split, hard in [('valid', {3, 4}), ('test', {5, 6})]: + dump(f'{split}-queries.pkl', queries) + dump(f'{split}-easy-answers.pkl', {q: {1, 7} for group in queries.values() for q in group}) + dump(f'{split}-hard-answers.pkl', {q: hard for group in queries.values() for q in group}) + + +@pytest.fixture(autouse=True) +def threads(): + previous = torch.get_num_threads() + torch.set_num_threads(2) + yield + torch.set_num_threads(previous) + + +def small_data(): + queries = tuple(BenchmarkQuery('1p', (i, (0,)), frozenset({0}), frozenset({1, 2})) for i in range(6)) + return QueryBenchmark('test', 'transductive', 'valid', QueryContext([(0, 0, 1)], 6, 1), queries, tuple(range(6))) + + +def test_source_identity_is_the_same_through_a_symlinked_checkout(tmp_path): + """A checkout imported through a symlink (for example a mounted home) keeps its source identity.""" + link = tmp_path / 'checkout' + link.symlink_to(SOURCE_ROOT, target_is_directory=True) + linked = subprocess.run([sys.executable, '-c', 'from benchmarks.cqa.study import code_identity; print(code_identity())'], + cwd=tmp_path, env=dict(os.environ, PYTHONPATH=str(link)), capture_output=True, text=True, check=True) + assert linked.stdout.strip() == code_identity() + + +def test_resume_preserves_completed_cache_report_and_resets_live_cache(tmp_path): + data, stats = small_data(), {} + options = dict(checkpoint_dir=tmp_path / 'resume', checkpoint_identity='fixed', checkpoint_every=3) + calls = 0 + + def predict(query): + nonlocal calls + calls += 1 + if calls == 5: + raise RuntimeError('interrupted') + stats['raw_rows'] = stats.get('raw_rows', 0) + 1 + stats.update(cache_bytes=256, cache_peak_bytes=256) + return torch.arange(6).float() + + with pytest.raises(RuntimeError, match='interrupted'): + evaluate_benchmark(data, predict, statistics=stats, **options) + stats, observations = {}, [] + + def resumed_predict(query): + stats['raw_rows'] += 1 + stats['cache_bytes'] = 128 + return torch.arange(6).float() + + report = evaluate_benchmark(data, resumed_predict, statistics=stats, + progress=lambda *_: observations.append(stats['cache_bytes']), **options) + assert observations[0] == 0 + assert stats == dict(raw_rows=6, cache_bytes=128, cache_peak_bytes=256) + replayed_stats = {} + replayed = evaluate_benchmark(data, lambda _: pytest.fail('Completed queries were rescored'), + statistics=replayed_stats, **options) + assert replayed['per_shape'] == report['per_shape'] + assert replayed_stats == stats + + +def test_trix_completed_resume_preserves_cache_statistics(tmp_path, monkeypatch): + from dicee.models import TRIX + from dicee.query_answering import QueryAnswerer, benchmark_model + + model = TRIX(dict(num_entities=1, num_relations=1, trix_dim=8)) + options = dict(beam_size=2, row_batch_size=4, checkpoint_dir=tmp_path / 'trix', checkpoint_every=3) + report = benchmark_model(model, small_data(), **options) + assert report['inference']['statistics']['cache_bytes'] > 0 + monkeypatch.setattr(QueryAnswerer, 'predict', lambda *a, **k: pytest.fail('Completed queries were rescored')) + resumed = benchmark_model(model, small_data(), **options) + assert resumed['per_shape'] == report['per_shape'] + assert resumed['inference']['statistics'] == report['inference']['statistics'] + assert model.query_batch_size == 8 and model.training and model.graph_triples is None + + +def test_kgfm_determinism_is_strict_and_restores_caller_state(): + from dicee.query_answering.method_evaluation import deterministic_kgfm + enabled = torch.are_deterministic_algorithms_enabled() + warning = torch.is_deterministic_algorithms_warn_only_enabled() + try: + torch.use_deterministic_algorithms(False, warn_only=True) + with pytest.raises(RuntimeError, match='probe'): + with deterministic_kgfm(): + assert torch.are_deterministic_algorithms_enabled() + assert not torch.is_deterministic_algorithms_warn_only_enabled() + raise RuntimeError('probe') + assert not torch.are_deterministic_algorithms_enabled() + assert torch.is_deterministic_algorithms_warn_only_enabled() + finally: + torch.use_deterministic_algorithms(enabled, warn_only=warning) + + +def test_one_sort_and_exact_uniform_tie_expectation(monkeypatch): + calls = 0 + original = torch.Tensor.argsort + + def counted(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(torch.Tensor, 'argsort', counted) + scores = torch.tensor([.8, .5, .5, .5, .2, .5]) + metrics, records = QueryMetrics(6, candidates=[0, 1, 2, 3, 4]).evaluate(scores, {0}, {1, 2}, records=True) + assert calls == 1 + assert [row[2:] for row in records] == [[0, 2], [0, 2]] + permutations = list(itertools.permutations([1, 2, 3])) + rr = [] + for order in permutations: + rr.append(np.mean([1 / (1 + sum(x == 3 for x in order[:order.index(target)])) for target in (1, 2)])) + assert metrics['expected']['mrr'] == pytest.approx(np.mean(rr)) + assert metrics['expected']['mrr'] == .75 + + +def test_trace_resume_replays_whole_batch_and_truncates_uncheckpointed_rows(tmp_path, monkeypatch): + data, prepared, seen = small_data(), {}, [] + + def prepare(queries): + seen.append(tuple(queries)) + values = torch.arange(6).float().remainder(1 + len(queries)) + prepared.update({query: values for query in queries}) + + options = dict(query_plan=data.queries, batch_ends=[3, 6], prepare=prepare, checkpoint_every=2, + checkpoint_dir=tmp_path / 'progress', checkpoint_identity={'model': 'batch-dependent'}, + rank_trace_path=tmp_path / 'ranks.sqlite3', additional_tie_policies=('expected',)) + original = BenchmarkCheckpoint.save + + def fail_after_trace_commit(self, state): + if state['completed'] == 6: + raise RuntimeError('power loss') + original(self, state) + + monkeypatch.setattr(BenchmarkCheckpoint, 'save', fail_after_trace_commit) + with pytest.raises(RuntimeError, match='power loss'): + evaluate_benchmark(data, lambda query: prepared[query], **options) + saved = read(tmp_path / 'progress' / 'state.json')['state'] + assert saved['completed'] == 3 + with sqlite3.connect(tmp_path / 'ranks.sqlite3') as connection: + assert connection.execute('SELECT COUNT(*) FROM queries').fetchone()[0] == 6 + monkeypatch.setattr(BenchmarkCheckpoint, 'save', original) + resumed = evaluate_benchmark(data, lambda query: prepared[query], **options) + assert [len(batch) for batch in seen] == [3, 3, 3] + assert seen[1] == seen[2] + rows = list(trace_queries(tmp_path / 'ranks.sqlite3')) + assert len(rows) == 6 and len({row['query_id'] for row in rows}) == 6 + summary = summarize_trace(tmp_path / 'ranks.sqlite3', bootstrap_samples=20) + assert summary['averages']['sort']['all']['mrr'] == resumed['averages']['all']['mrr'] + assert summary['averages']['expected']['all']['mrr'] == resumed['additional_tie_metrics']['expected']['averages']['all']['mrr'] + + +def paper_fixture(tmp_path): + data_root = tmp_path / 'data' + plus_h_fixture(data_root, 'FB15k237+H') + context = QueryContext([(0, 0, 1)], 8, 4) + model = ConE(context, dim=8, gamma=12, center_reg=.02, projection_dim=16) + torch.save(model.state_dict(), tmp_path / 'cone.pt') + entry = dict(id='cone-fixture', method='cone', dataset='FB15k237+H', checkpoint='cone.pt', + options={'center_reg': .02}, query_types=list(PLUS_H_SHAPES), query_batch_size=1, + query_order='upstream-pickle', reference={'status': 'test fixture'}, blockers=[]) + manifest = dict(version=1, data_root='data', seed=0, threads=2, archives={}, entries=[entry]) + return manifest + + +@pytest.mark.parametrize('name', ('FB15k237+H', 'NELL995+H', 'ICEWS18+H')) +def test_graph_conditions_keep_labels_fixed_and_never_read_test_edges(tmp_path, name): + from dicee.query_answering.datasets import load_benchmark + plus_h_fixture(tmp_path, name) + train = load_benchmark(tmp_path, name, inference_graph='train') + extended = load_benchmark(tmp_path, name, inference_graph='train+valid') + assert load_benchmark(tmp_path, name).context == extended.context + assert extended.metadata['inference_graph'] == 'train+valid' + assert set(train.context.triples) < set(extended.context.triples) + assert (0, 0, 3) not in train.context.triples and (0, 0, 3) in extended.context.triples + assert train.queries == extended.queries + assert [q.identity for q in train.queries] == [q.identity for q in extended.queries] + assert 'valid.txt' not in {Path(p).name for p in train.metadata['files']} + path = tmp_path / dataset_spec(name)[1] + graph = path / 'KG_splits' if name == 'ICEWS18+H' else path + (graph / 'test.txt').write_text('unreadable target triples') + assert load_benchmark(tmp_path, name, inference_graph='train+valid').context == extended.context + assert load_benchmark(tmp_path, name, split='valid', inference_graph='train+valid').context == train.context + assert load_benchmark(tmp_path, name, split='valid').context == train.context + + +def test_graph_conditions_pin_distinct_contexts_and_identical_query_plans(tmp_path): + from benchmarks.cqa.study import dataset_key + manifest = paper_fixture(tmp_path) + entry = manifest['entries'][0] + entry.update(inference_graph='train', graph_ablation='paired') + manifest['entries'].append(dict(entry, id='extended', inference_graph='train+valid')) + bundle = freeze(manifest, tmp_path / 'bundle', tmp_path) + other = manifest['entries'][1] + assert bundle['datasets'][dataset_key(entry, 'test')]['context'] != bundle['datasets'][dataset_key(other, 'test')]['context'] + assert dataset_key(entry, 'valid') == dataset_key(other, 'valid') + for split in ('valid', 'test'): + assert bundle['plans'][f'{entry["id"]}/{split}'] == bundle['plans'][f'{other["id"]}/{split}'] + assert len(bundle['datasets']) == 3 + + +def test_implicit_test_graph_matches_explicit_train_valid_bundle(tmp_path): + from benchmarks.cqa.study import dataset_key + manifest = paper_fixture(tmp_path) + entry = manifest['entries'][0] + explicit = dict(entry, id='explicit', inference_graph='train+valid') + manifest['entries'].append(explicit) + bundle = freeze(manifest, tmp_path / 'bundle', tmp_path) + assert dataset_key(entry, 'test') == dataset_key(explicit, 'test') == 'FB15k237+H/test/train+valid' + assert len(bundle['datasets']) == 2 + assert bundle['datasets'][dataset_key(entry, 'test')]['metadata']['inference_graph'] == 'train+valid' + assert bundle['plans'][f'{entry["id"]}/test'] == bundle['plans'][f'{explicit["id"]}/test'] + + +def test_direct_evaluation_rejects_supplied_wrong_graph(tmp_path): + from dicee.query_answering.datasets import load_benchmark + from dicee.query_answering.method_evaluation import evaluate_method + paper_fixture(tmp_path) + data = load_benchmark(tmp_path / 'data', 'FB15k237+H', inference_graph='train+valid') + with pytest.raises(ValueError, match='requested inference graph'): + evaluate_method(dict(method='cone', dataset='FB15k237+H', inference_graph='train'), + output=tmp_path / 'result', data=data) + + +def test_baseline_filter_control_keeps_inference_and_graph_metadata(tmp_path): + from dicee.query_answering.datasets import load_benchmark + from dicee.query_answering.method_evaluation import evaluate_method + manifest = paper_fixture(tmp_path) + data = load_benchmark(tmp_path / 'data', 'FB15k237+H', query_types=['2in'], inference_graph='train') + entry = manifest['entries'][0] + recipe = dict(method='cone', dataset=entry['dataset'], vocabulary_dataset=entry['dataset'], + root=str(tmp_path / 'data'), checkpoint=str(tmp_path / entry['checkpoint']), + options=entry['options'], query_types=['2in'], inference_graph='train') + report = evaluate_method(recipe, output=tmp_path / 'evaluation', data=data, filter_corrections={}) + control = read(tmp_path / 'evaluation/released-filters/result.json') + assert report['inference'] == control['inference'] + assert report['per_shape'] == control['per_shape'] + assert report['protocol']['answer_filter'] == 'corrected' + assert control['protocol']['answer_filter'] == 'released' + assert 'paired_with' not in control + assert control['filter_paired_with'] == report['benchmark_run']['entry'] + + +def test_filter_audit_preserves_targets_and_keeps_test_facts_out_of_context(tmp_path): + from dataclasses import replace + + from dicee.query_answering.datasets import audit_plus_h_filters + plus_h_fixture(tmp_path, 'FB15k237+H') + query = BenchmarkQuery('2in', ((0, (0,)), (1, (2, -2))), frozenset({1, 2, 3}), frozenset({5})) + other = BenchmarkQuery('3in', ((0, (0,)), (0, (0,)), (1, (2, -2))), frozenset({1, 3}), frozenset({5})) + folder = tmp_path / dataset_spec('FB15k237+H')[1] / 'test-query-reduction/2in/all' + folder.mkdir(parents=True) + for kind, value in [('queries', {QUERY_SHAPES['2in']: {query.query}}), + ('easy-answers', {query.query: {1, 3}}), ('hard-answers', {query.query: {5}})]: + (folder / f'test-{kind}.pkl').write_bytes(pickle.dumps(value)) + data = QueryBenchmark('FB15k237+H', 'transductive', 'test', QueryContext([(0, 0, 1)], 8, 4), + (query, other), tuple(range(8))) + audit = audit_plus_h_filters(tmp_path, data) + assert audit['changes'] == {query.identity: {'remove': [2], 'add': []}} + assert audit['per_shape']['3in']['changed_queries'] == 0 + assert audit['per_shape']['3in']['hard_answers'] == 1 + assert data.context.triples == ((0, 0, 1),) + invalid = replace(data, queries=(replace(query, hard=frozenset({6})),)) + with pytest.raises(ValueError, match='hard targets'): + audit_plus_h_filters(tmp_path, invalid) + (folder / 'test-easy-answers.pkl').write_bytes(pickle.dumps({query.query: {1, 2, 3}})) + with pytest.raises(ValueError, match='filters disagree with full-graph truth'): + audit_plus_h_filters(tmp_path, data) + + +def test_frozen_filter_corrections_reject_tampering(tmp_path, monkeypatch): + import dicee.query_answering.datasets as datasets + manifest = paper_fixture(tmp_path) + manifest['answer_filter'] = 'corrected' + monkeypatch.setattr(datasets, 'audit_plus_h_filters', lambda *args: dict(changes={}, per_shape={}, source_files={})) + bundle = freeze(manifest, tmp_path / 'bundle', tmp_path) + verify_bundle(tmp_path / 'bundle', tmp_path) + correction = bundle['filter_corrections']['FB15k237+H'] + (tmp_path / 'bundle' / correction['path']).write_text('{}') + with pytest.raises(ValueError, match='Answer-filter correction checksum'): + verify_bundle(tmp_path / 'bundle', tmp_path) + + +def use_ultraquery_fixture(manifest, root): + from dicee.query_answering._query import ULTRAQUERY_SHAPES + data = root / manifest['data_root'] + source = data / dataset_spec('FB15k237+H')[1] + destination = data / dataset_spec('FB15k237LogicalQuery')[1] + shutil.copytree(source, destination) + for split in ('valid', 'test'): + path = destination / f'{split}-queries.pkl' + queries = pickle.loads(path.read_bytes()) + path.write_bytes(pickle.dumps({QUERY_SHAPES[shape]: queries[QUERY_SHAPES[shape]] for shape in ULTRAQUERY_SHAPES})) + for entry in manifest['entries']: + entry.update(dataset='FB15k237LogicalQuery', query_types=list(ULTRAQUERY_SHAPES)) + + +def test_shared_study_and_reports_use_dataset_coverage(tmp_path): + from benchmarks.cqa.cli import main + from dicee.query_answering.method_evaluation import evaluate_method + manifest = paper_fixture(tmp_path) + use_ultraquery_fixture(manifest, tmp_path) + frozen = freeze(manifest, tmp_path / 'bundle', tmp_path) + assert len(frozen['plans']) == 2 + entry = manifest['entries'][0] + root = str(tmp_path / manifest['data_root']) + recipe = dict(method='cone', dataset=entry['dataset'], vocabulary_dataset=entry['dataset'], root=root, + checkpoint=str(tmp_path / entry['checkpoint']), options=entry['options'], query_types=entry['query_types']) + # A direct evaluation can be reported without requiring a +H recipe. + result = evaluate_method(recipe, output=tmp_path / 'direct', split='test') + assert result['queries'] == 14 and result['protocol']['all_benchmark_shapes'] + pilot = run_job(tmp_path / 'bundle', entry['id'], tmp_path, tmp_path / 'pilot') + assert pilot['queries'] == 14 and pilot['coverage']['complete_benchmark_types'] + assert not pilot['coverage']['complete_16_types'] + assert frozen['datasets']['FB15k237LogicalQuery/valid']['context'] == frozen['datasets']['FB15k237LogicalQuery/test']['context'] + assert result['benchmark_run']['mode'] == 'direct' + published = tmp_path / 'published.json' + write_json(published, {'source': 'fixture', 'datasets': {entry['dataset']: {'cone': dict.fromkeys(entry['query_types'], 10.)}}}) + rows = export_reports(tmp_path / 'direct', published, tmp_path / 'reports', bootstrap_samples=0) + assert rows[0]['complete_test'] and not rows[0]['complete_16_type_test'] + assert rows[0]['types'] == 14 and rows[0]['published_mrr'] == 10. + write_json(tmp_path / 'manifest.json', manifest) + main(['ultraquery', 'evaluate', '--split', 'test', '--manifests', str(tmp_path / 'manifest.json'), '--input-root', str(tmp_path), + '--output', str(tmp_path / 'cli'), '--entries', entry['id'], '--device', 'cpu']) + cli = read(tmp_path / 'cli' / entry['id'] / 'result.json') + assert cli['per_shape'] == result['per_shape'] + assert cli['additional_tie_metrics'] == result['additional_tie_metrics'] + + +@pytest.mark.parametrize('query_order', ('canonical', 'relation', 'upstream-pickle')) +def test_corrected_filter_cli_preserves_query_type_selection(tmp_path, query_order): + from benchmarks.cqa.cli import main + manifest = paper_fixture(tmp_path) + manifest['answer_filter'] = 'corrected' + entry = manifest['entries'][0] + entry.update(query_types=['1p'], query_order=query_order) + write_json(tmp_path / 'manifest.json', manifest) + main(['plus_h', 'evaluate', '--split', 'test', '--manifests', str(tmp_path / 'manifest.json'), '--input-root', str(tmp_path), + '--output', str(tmp_path / 'cli'), '--entries', entry['id'], '--device', 'cpu']) + destination = tmp_path / 'cli' / entry['id'] + result = read(destination / 'result.json') + control = read(destination / 'released-filters/result.json') + assert result['queries'] == control['queries'] == 1 + assert set(result['per_shape']) == set(control['per_shape']) == {'1p'} + assert result['per_shape'] == control['per_shape'] + assert result['protocol']['answer_filter'] == 'corrected' + assert control['protocol']['answer_filter'] == 'released' + + +def test_frozen_inputs_pipeline_pilot_and_offline_report(tmp_path): + manifest = paper_fixture(tmp_path) + bundle = tmp_path / 'bundle' + frozen = freeze(manifest, bundle, tmp_path) + assert len(frozen['datasets']) == 2 + assert len(verify_bundle(bundle, tmp_path)['files']) == 11 + output = tmp_path / 'results' / 'cone-fixture' + report = run_job(bundle, 'cone-fixture', tmp_path, output) + assert report['queries'] == 16 and report['split'] == 'valid' + assert report['protocol']['all_benchmark_shapes'] + assert run_job(bundle, 'cone-fixture', tmp_path, output) == report + published = tmp_path / 'published.json' + write_json(published, dict(source='fixture', datasets={'FB15k237+H': {'cone': dict.fromkeys(PLUS_H_SHAPES, 10.)}})) + rows = export_reports(tmp_path / 'results', published, tmp_path / 'tables', bootstrap_samples=10) + assert rows[0]['published_mrr'] is None + assert not rows[0]['complete_16_type_test'] + with pytest.raises(ValueError, match='parity evidence'): + run_job(bundle, 'cone-fixture', tmp_path, tmp_path / 'test', phase='test') + (tmp_path / 'cone.pt').write_bytes(b'changed') + with pytest.raises(ValueError, match='checksum mismatch'): + verify_bundle(bundle, tmp_path) + + +def test_plan_tampering_and_no_overwrite(tmp_path): + manifest = paper_fixture(tmp_path) + bundle = tmp_path / 'bundle' + frozen = freeze(manifest, bundle, tmp_path) + with pytest.raises(FileExistsError, match='new bundle'): + freeze(manifest, bundle, tmp_path) + plan = bundle / frozen['plans']['cone-fixture/test']['path'] + value = read(plan) + value['queries'].reverse() + write_json(plan, value) + with pytest.raises(ValueError, match='plan checksum'): + verify_bundle(bundle, tmp_path) + + +def test_probe_selection_preserves_full_encoding_batches(tmp_path, monkeypatch): + manifest = paper_fixture(tmp_path) + manifest['entries'][0]['query_batch_size'] = 4 + path = next((tmp_path / 'data').rglob('valid-queries.pkl')).parent + query = (4, (0,)) + for filename, update in ( + ('queries', lambda value: next(group for shape, group in value.items() if shape == ('e', ('r',))).add(query)), + ('easy-answers', lambda value: value.update({query: {1, 7}})), + ('hard-answers', lambda value: value.update({query: {3, 4}})), + ): + target = path / f'valid-{filename}.pkl' + value = pickle.loads(target.read_bytes()) + update(value) + target.write_bytes(pickle.dumps(value)) + calls = [] + original = ConE.encode_trees + + def coupled(self, trees): + calls.append(tuple(trees)) + return [branch + branch.mean(0, keepdim=True) for branch in original(self, trees)] + + monkeypatch.setattr(ConE, 'encode_trees', coupled) + bundle = tmp_path / 'bundle' + freeze(manifest, bundle, tmp_path) + full, probes = {}, {} + run_job(bundle, 'cone-fixture', tmp_path, tmp_path / 'full-pilot', pilot_queries=1, + on_prediction=lambda query, scores: full.update({query.query: scores.clone()})) + full_calls = calls.copy() + calls.clear() + report = run_job(bundle, 'cone-fixture', tmp_path, tmp_path / 'probe', pilot_queries=1, probe_only=True, + on_prediction=lambda query, scores: probes.update({query.query: scores.clone()})) + assert len(full) == 17 and len(probes) == report['queries'] == 16 + assert calls == full_calls + assert max(map(len, calls)) == 2 + assert all(torch.equal(score, full[query]) for query, score in probes.items()) + from dicee.query_answering.method_evaluation import evaluate_method + calls.clear() + direct = {} + entry = manifest['entries'][0] + recipe = dict(method='cone', checkpoint=str(tmp_path / entry['checkpoint']), + root=str(tmp_path / 'data'), dataset=entry['dataset'], vocabulary_dataset=entry['dataset'], + options=entry['options'], query_types=entry['query_types'], query_batch_size=4) + report = evaluate_method(recipe, output=tmp_path / 'direct', split='valid', limit=1, + query_order='upstream-pickle', + on_prediction=lambda query, scores: direct.update({query: scores.clone()})) + assert calls == full_calls and len(direct) == report['queries'] == 16 + assert all(torch.equal(score, full[query]) for query, score in direct.items()) + with pytest.raises(ValueError, match='validation pilots'): + run_job(bundle, 'cone-fixture', tmp_path, tmp_path / 'test', phase='test', probe_only=True) + + +def test_checked_in_protocol_settings(): + from benchmarks.cqa.manifests import read_manifest, suite_directory + root = suite_directory('plus_h') + manifest = read_manifest(root / 'baselines.json') + assert len(manifest['entries']) == 21 + by_id = {entry['id']: entry for entry in manifest['entries']} + assert by_id['cqd-FB15k237+H']['options']['per_shape']['2p']['beam_size'] == 512 + assert by_id['cqd-FB15k237+H']['options']['per_shape']['pi']['beam_size'] == 256 + assert by_id['cqd-hybrid-ICEWS18+H']['options']['per_shape']['4p']['max_k'] == 64 + assert by_id['cqd-FB15k237+H']['options']['reference_batching'] + for method in ('cqd', 'cqd-hybrid'): + assert by_id[f'{method}-FB15k237+H']['options']['per_shape']['2u']['tnorm'] == 'prod' + assert by_id['cone-ICEWS18+H']['query_batch_size'] == 4 + assert by_id['gnnqe-NELL995+H']['query_batch_size'] == 8 + assert by_id['ultraquery-FB15k237+H']['query_batch_size'] == 32 + assert by_id['clmpt-FB15k237+H']['query_batch_size'] == 5000 + assert by_id['clmpt-NELL995+H']['query_batch_size'] == 8 + for method in ('cqd', 'cqd-hybrid'): + for dataset in ('FB15k237+H', 'ICEWS18+H', 'NELL995+H'): + entry = by_id[f'{method}-{dataset}'] + assert len(entry['query_types']) == 16 + assert entry['options']['atomic_negation'] + published = read(root / 'published_results.json') + assert sum(published['datasets']['FB15k237+H']['ultraquery'].values()) / 16 == pytest.approx(9.6) + + +def test_rank_trace_rejects_missing_committed_answers(tmp_path): + data = small_data() + options = dict(checkpoint_dir=tmp_path / 'progress', checkpoint_identity={'a': 1}, rank_trace_path=tmp_path / 'ranks.sqlite3') + evaluate_benchmark(data, lambda query: torch.zeros(6), **options) + with sqlite3.connect(tmp_path / 'ranks.sqlite3') as connection: + connection.execute('DELETE FROM queries WHERE position=2') + with pytest.raises(ValueError, match='missing checkpointed'): + evaluate_benchmark(data, lambda query: torch.zeros(6), **options) + + +def test_invalid_batch_boundaries(tmp_path): + data = small_data() + for ends in ([2, 4], [3, 3, 6], [0, 6], [6, 3]): + with pytest.raises(ValueError, match='batch boundaries'): + evaluate_benchmark(data, lambda query: torch.zeros(6), query_plan=data.queries, batch_ends=ends) + + +@pytest.mark.integration +@pytest.mark.skipif(not os.environ.get('DICEE_CONE_REFERENCE'), reason='Set DICEE_CONE_REFERENCE to the pinned upstream checkout') +def test_independent_exporter_parity_and_full_run(tmp_path): + manifest = paper_fixture(tmp_path) + bundle = tmp_path / 'bundle' + freeze(manifest, bundle, tmp_path) + reference = tmp_path / 'reference.pt' + subprocess.run([sys.executable, str(Path(__file__).parents[1] / 'verification' / 'export_reference.py'), + '--bundle', str(bundle), '--input-root', str(tmp_path), '--entry', 'cone-fixture', + '--upstream', os.environ['DICEE_CONE_REFERENCE'], '--output', str(reference)], check=True) + evidence = verify_predictions(bundle, 'cone-fixture', tmp_path, reference, tmp_path / 'parity.json') + assert evidence['passed'] and len(evidence['queries']) == 16 + manifest['entries'][0]['verification'] = 'parity.json' + freeze(manifest, tmp_path / 'verified-bundle', tmp_path) + report = run_job(tmp_path / 'verified-bundle', 'cone-fixture', tmp_path, tmp_path / 'full', phase='test') + assert report['protocol']['full_split'] and report['coverage']['complete_16_types'] + assert report['split'] == 'test' + + +@pytest.mark.parametrize('backbone', ['ultra', 'trix', 'kgicl']) +def test_provisional_kgfm_pipeline_and_finalization_guard(tmp_path, backbone): + from dicee.models import KGICL, TRIX, ULTRA + manifest = paper_fixture(tmp_path) + model = {'ultra': ULTRA, 'trix': TRIX, 'kgicl': KGICL}[backbone](dict(num_entities=1, num_relations=1)) + # KG-ICL checkpoints keep their weights under 'state_dict'. + torch.save({'state_dict' if backbone == 'kgicl' else 'model': model.state_dict()}, tmp_path / 'kgfm.pt') + for operator in ('product', 'min'): + adapter = QueryScoreAdapter('global', observed_mix=1, metadata=dict( + backbone_state_sha256=state_fingerprint(model), training={} if operator == 'product' else {'tnorm': 'min'})) + adapter.save(tmp_path / f'{operator}.json') + entry = manifest['entries'][0] + entry.update(id=f'{backbone}-fixture', method=f'{backbone}-adapter', checkpoint='kgfm.pt', options={'beam_size': 2, 'cache_bytes': 4096}, + adapters={operator: f'{operator}.json' for operator in ('product', 'min')}, + operators={shape: 'min' if 'n' in shape else 'product' for shape in PLUS_H_SHAPES}, + selection_protocol='source-validation', finalized=False, query_batch_size=4, query_order='relation') + freeze(manifest, tmp_path / 'bundle', tmp_path) + result = run_job(tmp_path / 'bundle', entry['id'], tmp_path, tmp_path / 'pilot') + assert result['queries'] == 16 + assert result['inference']['selection_protocol'] == 'source-validation' + assert len(list(trace_queries(tmp_path / 'pilot' / 'ranks.sqlite3'))) == 16 + with pytest.raises(ValueError, match='not ready'): + run_job(tmp_path / 'bundle', entry['id'], tmp_path, tmp_path / 'test', phase='test') + + +def test_comparison_does_not_relax_reference_verification(tmp_path, monkeypatch): + import benchmarks.cqa.oracles as verification + from benchmarks.cqa.study import entry_inputs_identity + from dicee.query_answering.context import fingerprint + from dicee.query_answering.datasets import load_benchmark + from dicee.query_answering.methods import REFERENCES + manifest = paper_fixture(tmp_path) + entry = manifest['entries'][0] + entry['query_types'] = ['1p'] + entry['inference_graph'] = 'train' + bundle = freeze(manifest, tmp_path / 'bundle', tmp_path) + query = next(q for q in load_benchmark(tmp_path / 'data', 'FB15k237+H', split='valid').queries if q.shape == '1p') + key = query.identity + scores = torch.arange(8).float() + oracle = dict(version=1, entry_sha256=fingerprint(entry), inputs_sha256=entry_inputs_identity(bundle, entry), + validation_plan_sha256=bundle['plans'][entry['id'] + '/valid']['sha256'], + reference_commit=REFERENCES['cone'][1], environment=dict.fromkeys( + ('python', 'torch', 'cuda', 'gpu', 'driver', 'precision', 'threads'), 'fixture'), + graphs={split: bundle['datasets'][f'{entry["dataset"]}/{split}']['context'] for split in ('valid', 'test')}, + pilot_queries=1, probe_only=True, scores={key: scores}, orders={key: scores.argsort(descending=True)}) + torch.save(oracle, tmp_path / 'declared.pt') + previous = scores.clone() + previous[3] = 20 + control = dict(oracle, scores={key: previous}, orders={key: previous.argsort(descending=True)}) + torch.save(control, tmp_path / 'dense.pt') + + def run(*args, on_prediction, **kwargs): + on_prediction(query, scores) + return {'benchmark_run': {'environment': oracle['environment']}} + + monkeypatch.setattr(verification, 'run_job', run) + result = verify_predictions(tmp_path / 'bundle', entry['id'], tmp_path, tmp_path / 'declared.pt', + tmp_path / 'parity.json', comparison_reference=tmp_path / 'dense.pt') + assert result['passed'] + comparison = result['upstream_comparison'] + assert not comparison['queries'][key]['ranks_match'] + assert comparison['delta_macro']['sort']['mrr'] != 0 + assert comparison['delta_macro']['expected']['mrr'] != 0 + scores = previous + failed = verify_predictions(tmp_path / 'bundle', entry['id'], tmp_path, tmp_path / 'declared.pt', + tmp_path / 'failed.json', comparison_reference=tmp_path / 'dense.pt') + assert not failed['passed'] + control['inputs_sha256'] = 'different inputs' + torch.save(control, tmp_path / 'dense.pt') + with pytest.raises(ValueError, match='Comparison reference uses different'): + verify_predictions(tmp_path / 'bundle', entry['id'], tmp_path, tmp_path / 'declared.pt', + tmp_path / 'invalid.json', comparison_reference=tmp_path / 'dense.pt') + oracle['graphs']['test'] = 'wrong graph' + torch.save(oracle, tmp_path / 'wrong-graph.pt') + with pytest.raises(ValueError, match='Reference inference graphs differ'): + verify_predictions(tmp_path / 'bundle', entry['id'], tmp_path, tmp_path / 'wrong-graph.pt', + tmp_path / 'wrong-graph.json') + + +def test_suite_effects_average_seeds_per_query_and_pair_the_shared_control(tmp_path): + from collections import namedtuple + + from benchmarks.cqa.reports import suite_effects + from dicee.query_answering._rank_trace import RankTrace + from dicee.query_answering.catalog import PLUS_H_DATASETS + Query = namedtuple('Query', 'identity shape') + records = [] + runs = (('ultra-r-{}', 'ultra-adapter', 1, {}), ('ultra-r-seed1-{}', 'ultra-adapter', 2, {}), + ('ultra-r-{}-without-adapter', 'ultra-adapter', 4, dict(paired_with='ultra-r-{}', calibration='without-adapter')), + ('ultraquery-{}', 'ultraquery', 2, {})) + for dataset in PLUS_H_DATASETS: + for template, method, rank, extra in runs: + entry = template.format(dataset) + path = tmp_path / entry / 'ranks.sqlite3' + with RankTrace(path, {'entry': entry}, 0) as trace: + for position, shape in enumerate(['1p'] * 4 + ['2in'] * 4): + trace.add(position, Query(f'{dataset}-{position}', shape), [[7, rank, rank - 1, 1]]) + trace.commit() + records.append(dict(id=entry, dataset=dataset, method=method, complete=True, answer_filter='corrected', trace=path, + paired_with=extra.get('paired_with', '').format(dataset) or None, + calibration=extra.get('calibration', 'learned'))) + effects = {e['comparison']: e for e in suite_effects(records, bootstrap_samples=20)} + assert set(effects) == {'adapter-minus-identity', 'adapter-minus-ultraquery', 'identity-minus-ultraquery'} + # Seed tags 0 and 1 rank the answer first and second: a per-query seed mean of 0.75 MRR. + for comparison, delta in (('adapter-minus-identity', 50), ('adapter-minus-ultraquery', 25), ('identity-minus-ultraquery', -25)): + effect = effects[comparison] + assert effect['complete_suite'] and effect['datasets'] == 3 + for group in ('all', 'epfo', 'negation'): + assert effect['macro']['sort'][group]['delta'] == pytest.approx(delta) + assert effect['macro']['sort'][group]['ci95'] == pytest.approx([delta, delta]) + assert [s['sort']['all'] for s in effects['adapter-minus-identity']['per_seed']] == pytest.approx([75, 25]) + assert effects['adapter-minus-identity']['seed_tags'] == [0, 1] + # Only seeds covering every dataset enter the seed mean; a missing baseline run makes the suite incomplete. + records[0]['complete'] = False + effects = {e['comparison']: e for e in suite_effects(records, bootstrap_samples=0)} + assert effects['adapter-minus-identity']['seed_tags'] == [1] and effects['adapter-minus-identity']['complete_suite'] + next(r for r in records if r['method'] == 'ultraquery')['complete'] = False + effects = {e['comparison']: e for e in suite_effects(records, bootstrap_samples=0)} + assert effects['adapter-minus-ultraquery']['datasets'] == 2 and not effects['adapter-minus-ultraquery']['complete_suite'] + + +def test_reference_exemptions_are_exactly_the_negated_types_of_cqd_with_cqda_negation(): + import importlib.util + + import benchmarks.cqa.oracles as verification + spec = importlib.util.spec_from_file_location('export_reference', Path(__file__).parents[1] / 'verification/export_reference.py') + exporter = importlib.util.module_from_spec(spec) + spec.loader.exec_module(exporter) + negated = ['2in', '3in', 'inp', 'pin', 'pni'] + cases = [(dict(method='cqd', options={'atomic_negation': True}, query_types=list(PLUS_H_SHAPES)), negated), + (dict(method='cqd-hybrid', options={'atomic_negation': True}, query_types=['1p', '2in', 'pni']), ['2in', 'pni']), + (dict(method='cqd', options={}, query_types=list(PLUS_H_SHAPES)), []), + (dict(method='qto', options={'atomic_negation': True}, query_types=list(PLUS_H_SHAPES)), [])] + for entry, expected in cases: + assert verification.upstream_exempt_types(entry) == exporter.exempt_types(entry) == expected + assert exporter.EXEMPTION_REASON == verification.EXEMPTION_REASON + + +def test_exempt_types_are_evaluated_but_not_compared_and_must_be_declared(tmp_path, monkeypatch): + import benchmarks.cqa.oracles as verification + from benchmarks.cqa.study import entry_inputs_identity + from dicee.query_answering.context import fingerprint + from dicee.query_answering.datasets import load_benchmark + from dicee.query_answering.methods import REFERENCES + manifest = paper_fixture(tmp_path) + entry = manifest['entries'][0] + entry['query_types'] = ['1p', '2in'] + entry['inference_graph'] = 'train' + bundle = freeze(manifest, tmp_path / 'bundle', tmp_path) + queries = load_benchmark(tmp_path / 'data', 'FB15k237+H', split='valid').queries + checked, exempt = (next(q for q in queries if q.shape == shape) for shape in ('1p', '2in')) + scores = torch.arange(8).float() + oracle = dict(version=1, entry_sha256=fingerprint(entry), inputs_sha256=entry_inputs_identity(bundle, entry), + validation_plan_sha256=bundle['plans'][entry['id'] + '/valid']['sha256'], + reference_commit=REFERENCES['cone'][1], environment=dict.fromkeys( + ('python', 'torch', 'cuda', 'gpu', 'driver', 'precision', 'threads'), 'fixture'), + graphs={split: bundle['datasets'][f'{entry["dataset"]}/{split}']['context'] for split in ('valid', 'test')}, + pilot_queries=1, probe_only=True, scores={checked.identity: scores}, + orders={checked.identity: scores.argsort(descending=True)}, + exempt_types=dict(types=['2in'], reason=verification.EXEMPTION_REASON)) + torch.save(oracle, tmp_path / 'oracle.pt') + + def run(*args, on_prediction, **kwargs): + on_prediction(checked, scores) + on_prediction(exempt, torch.zeros(8)) + return {'benchmark_run': {'environment': oracle['environment']}} + + monkeypatch.setattr(verification, 'run_job', run) + monkeypatch.setattr(verification, 'upstream_exempt_types', lambda entry: ['2in']) + result = verification.verify_predictions(tmp_path / 'bundle', entry['id'], tmp_path, tmp_path / 'oracle.pt', tmp_path / 'parity.json') + assert result['passed'] and set(result['queries']) == {checked.identity} + assert result['exempt_types']['types'] == ['2in'] and result['exempt_types']['evaluated_queries'] == {'2in': 1} + # An oracle has to declare exactly the exemptions the rule allows. + torch.save({key: value for key, value in oracle.items() if key != 'exempt_types'}, tmp_path / 'undeclared.pt') + with pytest.raises(ValueError, match='Reference exemptions differ'): + verification.verify_predictions(tmp_path / 'bundle', entry['id'], tmp_path, tmp_path / 'undeclared.pt', tmp_path / 'x.json') + monkeypatch.setattr(verification, 'upstream_exempt_types', lambda entry: []) + with pytest.raises(ValueError, match='Reference exemptions differ'): + verification.verify_predictions(tmp_path / 'bundle', entry['id'], tmp_path, tmp_path / 'oracle.pt', tmp_path / 'y.json') diff --git a/benchmarks/cqa/tests/test_ultraquery_launcher.py b/benchmarks/cqa/tests/test_ultraquery_launcher.py new file mode 100644 index 000000000..c2a39d0e5 --- /dev/null +++ b/benchmarks/cqa/tests/test_ultraquery_launcher.py @@ -0,0 +1,242 @@ +"""Shared launch interface, independent input reading, and final-run parity gates.""" + +import json +import os +import pickle +import subprocess +import sys + +import pytest +import torch + +from benchmarks.cqa import cli, docker +from benchmarks.cqa.manifests import REPO, SUITES, prepare_manifest, read_manifest, suite_directory, validate_entry +from benchmarks.cqa.oracles import verify_predictions +from benchmarks.cqa.study import dataset_key, entry_inputs_identity, freeze, run_job +from benchmarks.cqa.verification import export_reference as exporter +from dicee.query_answering import BENCHMARK_DATASETS, QueryContext, load_benchmark +from dicee.query_answering._query import QUERY_SHAPES, ULTRAQUERY_SHAPES +from dicee.query_answering.context import fingerprint +from dicee.query_answering.datasets import dataset_spec +from dicee.query_answering.methods import REFERENCES, UltraQuery + +PUBLIC = suite_directory('ultraquery') + + +@pytest.fixture(autouse=True) +def threads(): + previous = torch.get_num_threads() + torch.set_num_threads(2) + yield + torch.set_num_threads(previous) + + +def fixture(root, name): + group, folder, _ = dataset_spec(name) + path = root / 'data' / folder + path.mkdir(parents=True) + pairs = ((0, 1), (2, 3)) if group == 'transductive' else ((0, 2), (1, 3)) + + def graph(stem, direct): + triples = QueryContext(direct, 8, 4, pairs).triples + (path / f'{stem}.txt').write_text(''.join(f'{h}\t{r}\t{t}\n' for h, r, t in triples)) + + def dump(filename, value): + (path / filename).write_bytes(pickle.dumps(value)) + + train = [(0, 0, 1), (1, 2 if group == 'transductive' else 1, 2), (2, 0, 3)] + if group == 'transductive': + graph('train', train) + dump('id2ent.pkl', {i: str(i) for i in range(8)}) + dump('id2rel.pkl', {i: str(i) for i in range(4)}) + else: + graph('train_graph', train) + if group == 'inductive-e': + graph('val_inference', [(2, 1, 5)]) + graph('test_inference', [(2, 1, 6)]) + else: + graph('test_inference', [(0, 1, 3), (3, 0, 1), (1, 1, 2)]) + + def instantiate(shape, counts): + if shape in ('n', 'u'): + return -2 if shape == 'n' else -1 + if shape in ('e', 'r'): + i = int(shape == 'r') + value = counts[i] + counts[i] += 1 + return value % (4 if i else 8) + return tuple(instantiate(part, counts) for part in shape) + + queries = {QUERY_SHAPES[s]: {instantiate(QUERY_SHAPES[s], [0, 0])} for s in ULTRAQUERY_SHAPES} + for split in ('valid', 'test'): + if group == 'transductive': + dump(f'{split}-queries.pkl', queries) + dump(f'{split}-easy-answers.pkl', {q: {1} for qs in queries.values() for q in qs}) + dump(f'{split}-hard-answers.pkl', {q: {3} for qs in queries.values() for q in qs}) + else: + dump(f'{split}_queries.pkl', queries) + dump(f'{split}_answers_easy.pkl', {s: {q: {1} for q in qs} for s, qs in queries.items()}) + dump(f'{split}_answers_hard.pkl', {s: {q: {3} for q in qs} for s, qs in queries.items()}) + data = load_benchmark(root / 'data', name, split='valid') + model = UltraQuery(data.context, dim=8, num_layers=2) + torch.save({'model': {'model.model.' + k: v for k, v in model.state_dict().items()}}, root / 'model.pt') + entry = dict(id='ultraquery-fixture', method='ultraquery', dataset=name, checkpoint='model.pt', + options={'threshold': 0., 'logic': 'product'}, query_types=list(ULTRAQUERY_SHAPES), + query_batch_size=32, query_order='upstream-pickle', reference={'status': 'test fixture'}, blockers=[]) + return dict(version=1, data_root='data', entries=[entry], seed=0, threads=2, archives={}, answer_filter='released') + + +def test_public_recipes_cover_all_datasets_and_selected_adapters(): + paths = [PUBLIC / name for name in ('baselines.json', 'kgfm_adapters.json')] + manifest = prepare_manifest(paths, suite='ultraquery', answer_filter='released') + assert len(manifest['entries']) == 69 + assert all('-product-intersections-' in e['id'] for e in manifest['entries'] if e['method'].endswith('-adapter')) + for entry in manifest['entries']: + validate_entry(entry) + assert set(entry['query_types']) == set(ULTRAQUERY_SHAPES) + assert 'inference_graph' not in entry + for method in ('ultraquery', 'ultra-adapter', 'trix-adapter'): + assert {e['dataset'] for e in manifest['entries'] if e['method'] == method} == set(BENCHMARK_DATASETS) + for backbone in ('ultra', 'trix'): + adapter = json.loads((REPO / 'benchmarks/adapters' / f'{backbone}_product_intersections.json').read_text()) + assert adapter['training']['shapes'] == ['2i', '3i'] + # Both suites share one copy of every released adapter. + shared = {path for suite in SUITES for name in ('kgfm_adapters.json',) + if (suite_directory(suite) / name).is_file() + for entry in read_manifest(suite_directory(suite) / name)['entries'] for path in entry['adapters'].values()} + assert shared and all(path.startswith('benchmarks/adapters/') and (REPO / path).is_file() for path in shared) + + +def test_cli_subset_and_suite_policies_fail_before_opening_inputs(tmp_path, capsys): + cli.main(['ultraquery', 'evaluate', '--split', 'test', '--methods', 'ultraquery', 'ultra-adapter', '--datasets', 'WikiTopicsQuery:art', + 'InductiveFB15k237Query:106', '--query-types', 'negation', '--input-root', str(tmp_path), + '--output', str(tmp_path / 'results'), '--dry-run']) + manifest = json.loads(capsys.readouterr().out) + assert len(manifest['entries']) == 4 and manifest['answer_filter'] == 'released' + assert all(set(e['query_types']) == {'2in', '3in', 'inp', 'pin', 'pni'} for e in manifest['entries']) + assert not (tmp_path / 'results').exists() + for selection in (['--answer-filter', 'corrected'], ['--datasets', 'FB15k237+H'], ['--methods', 'bad']): + with pytest.raises(SystemExit): + cli.main(['ultraquery', 'evaluate', '--split', 'test', '--output', str(tmp_path / 'results'), '--dry-run', *selection]) + + +@pytest.mark.parametrize('action', ['prepare', 'evaluate', 'run', 'report']) +def test_docker_actions_use_the_shared_runtime_and_correct_suite(tmp_path, capsys, action): + args = ['ultraquery', action, '--image', 'dicee/cqa:test', '--input-root', str(tmp_path), '--output', str(tmp_path / 'results'), + '--gpu', 'none', '--dry-run'] + if action in ('evaluate', 'run'): + args += ['--device', 'cpu'] + if action == 'evaluate': + args += ['--split', 'test'] + if action == 'report': + args += ['--results', str(tmp_path / 'results/test')] + else: + args += ['--methods', 'ultraquery', '--datasets', 'WikiTopicsQuery:art'] + if action in ('evaluate', 'prepare'): + args += ['--query-types', 'negation'] + cli.main(args) + output = capsys.readouterr().out + assert f'dicee/cqa:test ultraquery {action} --input-root /inputs --output /results' in output + assert '--gpus' not in output and '--published' not in output + if action == 'run': + with pytest.raises(SystemExit): + cli.main([*args, '--query-types', '1p']) + assert not (tmp_path / 'results').exists() + + +def test_docker_custom_manifests_are_mapped_and_confined_to_inputs(tmp_path, capsys): + inputs = tmp_path / 'inputs' + inputs.mkdir() + manifests = [inputs / name for name in ('native.json', 'adapters.json')] + for path in manifests: + path.write_text('{}') + args = ['ultraquery', 'prepare', '--image', 'dicee/cqa:test', '--input-root', str(inputs), '--output', str(inputs / 'results'), '--dry-run'] + cli.main([*args, '--manifests', str(manifests[0]), '--manifests', str(manifests[1])]) + assert '--manifests /inputs/native.json /inputs/adapters.json' in capsys.readouterr().out + outside = tmp_path / 'outside.json' + outside.write_text('{}') + with pytest.raises(SystemExit): + cli.main([*args, '--manifests', str(outside)]) + assert 'below --input-root' in capsys.readouterr().err + assert not (inputs / 'results').exists() + + +def test_container_runs_the_cli_entrypoint_with_a_pinned_image(tmp_path, monkeypatch): + launches = [] + monkeypatch.setattr(docker.subprocess, 'check_output', lambda *args, **kwargs: '[{"Id":"sha256:test"}]') + monkeypatch.setattr(docker.subprocess, 'run', lambda command, **kwargs: launches.append(command)) + results = tmp_path / 'results' + results.mkdir() + docker.run('dicee/cqa:test', ['ultraquery', 'evaluate', '--output', '/results'], inputs=tmp_path, results=results) + command = launches[0] + assert '--entrypoint' not in command and command[command.index('--network') + 1] == 'none' + assert command[command.index('sha256:test') + 1:] == ['ultraquery', 'evaluate', '--output', '/results'] + assert json.loads((results / 'container-image.json').read_text())['image_id'] == 'sha256:test' + + +@pytest.mark.parametrize('name', ['FB15k237LogicalQuery', 'InductiveFB15k237Query:106', 'WikiTopicsQuery:art']) +def test_independent_exporter_reads_all_three_published_layouts(tmp_path, name): + manifest = fixture(tmp_path, name) + bundle = freeze(manifest, tmp_path / 'bundle', tmp_path) + prefix, structured, easy, hard, graph, graphs = exporter.reference_dataset(bundle, manifest['entries'][0], tmp_path) + assert set(graph['triples']) == set(load_benchmark(tmp_path / 'data', name, split='valid').context.triples) + assert {q for qs in structured.values() for q in qs} == set(easy) == set(hard) + assert graphs == {split: bundle['datasets'][dataset_key(manifest['entries'][0], split)]['context'] for split in ('valid', 'test')} + assert prefix.endswith(dataset_spec(name)[1] + '/') + + +def test_final_runs_require_matching_parity_and_preserve_14_type_coverage(tmp_path): + manifest = fixture(tmp_path, 'WikiTopicsQuery:art') + bundle = freeze(manifest, tmp_path / 'bundle', tmp_path) + with pytest.raises(ValueError, match='requires pinned reference'): + run_job(tmp_path / 'bundle', 'ultraquery-fixture', tmp_path, tmp_path / 'test', phase='test') + scores = {} + pilot = run_job(tmp_path / 'bundle', 'ultraquery-fixture', tmp_path, tmp_path / 'pilot', + on_prediction=lambda q, values: scores.update({q.identity: values.clone()})) + entry = manifest['entries'][0] + oracle = dict(version=1, entry_sha256=fingerprint(entry), inputs_sha256=entry_inputs_identity(bundle, entry), + validation_plan_sha256=bundle['plans']['ultraquery-fixture/valid']['sha256'], + reference_commit=REFERENCES['ultraquery'][1], pilot_queries=2, scores=scores, + orders={k: v.argsort(descending=True) for k, v in scores.items()}, + environment=pilot['benchmark_run']['environment'], + graphs={split: bundle['datasets'][dataset_key(entry, split)]['context'] for split in ('valid', 'test')}) + reference = tmp_path / 'reference.pt' + torch.save(oracle, reference) + evidence = verify_predictions(tmp_path / 'bundle', entry['id'], tmp_path, reference, tmp_path / 'evidence.json') + assert evidence['passed'] and len(evidence['queries']) == 14 + missing_graphs = dict(oracle, graphs={}) + torch.save(missing_graphs, reference) + with pytest.raises(ValueError, match='inference graphs'): + verify_predictions(tmp_path / 'bundle', entry['id'], tmp_path, reference, tmp_path / 'bad.json') + oracle['scores'][next(iter(scores))] = torch.zeros_like(next(iter(scores.values()))) + oracle['orders'] = {k: v.argsort(descending=True) for k, v in oracle['scores'].items()} + torch.save(oracle, reference) + assert not verify_predictions(tmp_path / 'bundle', entry['id'], tmp_path, reference, tmp_path / 'failed.json')['passed'] + entry['verification'] = 'evidence.json' + freeze(manifest, tmp_path / 'verified', tmp_path) + report = run_job(tmp_path / 'verified', entry['id'], tmp_path, tmp_path / 'test', phase='test') + assert report['protocol']['full_split'] and report['coverage']['complete_benchmark_types'] + assert not report['coverage']['complete_16_types'] + assert set(report['per_shape']) == set(ULTRAQUERY_SHAPES) + assert set(report['additional_tie_metrics']) == {'expected'} + cli.main(['ultraquery', 'report', '--results', str(tmp_path / 'test'), '--output', str(tmp_path / 'reports'), '--bootstrap-samples', '0']) + rows = json.loads((tmp_path / 'reports/comparison.json').read_text())['rows'] + assert rows[0]['complete_test'] and rows[0]['types'] == 14 and rows[0]['published_mrr'] is None + + +@pytest.mark.integration +@pytest.mark.skipif(not os.environ.get('DICEE_ULTRAQUERY_REFERENCE_ROOT'), reason='Set the pinned upstream root and dependencies') +@pytest.mark.parametrize('name', ['FB15k237LogicalQuery', 'InductiveFB15k237Query:106', 'WikiTopicsQuery:art']) +def test_independent_ultraquery_export_parity_and_full_run(tmp_path, name): + manifest = fixture(tmp_path, name) + freeze(manifest, tmp_path / 'bundle', tmp_path) + reference = tmp_path / 'reference.pt' + subprocess.run([sys.executable, str(REPO / 'benchmarks/cqa/verification/export_reference.py'), '--bundle', str(tmp_path / 'bundle'), + '--input-root', str(tmp_path), '--entry', 'ultraquery-fixture', '--upstream', + os.environ['DICEE_ULTRAQUERY_REFERENCE_ROOT'], '--output', str(reference)], check=True) + evidence = verify_predictions(tmp_path / 'bundle', 'ultraquery-fixture', tmp_path, reference, tmp_path / 'evidence.json') + assert evidence['passed'] and len(evidence['queries']) == 14 + manifest['entries'][0]['verification'] = 'evidence.json' + freeze(manifest, tmp_path / 'verified', tmp_path) + report = run_job(tmp_path / 'verified', 'ultraquery-fixture', tmp_path, tmp_path / 'test', phase='test') + assert report['protocol']['full_split'] and report['coverage']['complete_benchmark_types'] diff --git a/benchmarks/cqa/tests/test_ultraquery_protocol.py b/benchmarks/cqa/tests/test_ultraquery_protocol.py new file mode 100644 index 000000000..4f28612bd --- /dev/null +++ b/benchmarks/cqa/tests/test_ultraquery_protocol.py @@ -0,0 +1,164 @@ +"""Optional conformance checks against the released UltraQuery suite and code.""" + +import hashlib +import os +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pytest +import torch + +from dicee.query_answering import BENCHMARK_DATASETS, evaluate_benchmark, load_benchmark +from dicee.query_answering._query import ULTRAQUERY_SHAPES +from dicee.query_answering.benchmark import QueryMetrics +from dicee.query_answering.datasets import REFERENCE_COMMIT, dataset_spec + + +@pytest.fixture(autouse=True) +def threads(): + previous = torch.get_num_threads() + torch.set_num_threads(2) + yield + torch.set_num_threads(previous) + + +@pytest.fixture +def upstream_reference(monkeypatch): + if not os.environ.get('DICEE_ULTRAQUERY_REFERENCE_ROOT'): + pytest.skip('Set DICEE_ULTRAQUERY_REFERENCE_ROOT, with upstream dependencies installed') + reference = Path(os.environ['DICEE_ULTRAQUERY_REFERENCE_ROOT']) + # Verify the actual source files, including when a Docker image lacks git. + hashes = { + 'query_utils.py': '9b41775a5b54a0e40b9bd3acc24e0f20e54ecf0150f30beae382aa95c36191a0', + 'datasets_query.py': 'c82a04f477724cf3b6d69ef9e2bcdc2c03b7da0b856c47c915d8b3608168a510', + } + for name, checksum in hashes.items(): + assert hashlib.sha256((reference / 'ultra' / name).read_bytes()).hexdigest() == checksum, REFERENCE_COMMIT + monkeypatch.syspath_prepend(str(reference)) + from ultra import datasets_query, query_utils + return datasets_query, query_utils + + +def test_published_negative_infinity_candidate_edge_case(upstream_reference): + _, reference = upstream_reference + scores = torch.full((1, 6), -torch.inf) + easy = torch.zeros_like(scores, dtype=torch.bool) + hard = easy.clone() + hard[0, 2] = True + candidates = torch.tensor([0, 2, 4]) + published, _ = reference.batch_evaluate(scores.clone(), (torch.tensor([0]), easy, hard), candidates) + _, records = QueryMetrics(6, candidates=candidates.tolist()).evaluate(scores[0], set(), {2}, records=True) + assert published.tolist() == [3] + assert records[0][1] == 2 + + +@pytest.mark.integration +@pytest.mark.skipif(not os.environ.get('DICEE_ULTRAQUERY_DATA_ROOT'), reason='Set DICEE_ULTRAQUERY_DATA_ROOT to the official extracted data') +@pytest.mark.parametrize('name', BENCHMARK_DATASETS) +@pytest.mark.parametrize('split', ['valid', 'test']) +def test_official_ultraquery_data_graphs_and_reference_ranks(name, split): + """Real-data conformance for all 23 datasets in the published suite.""" + root = Path(os.environ['DICEE_ULTRAQUERY_DATA_ROOT']) + data = load_benchmark(root, name, split=split) + path = root / dataset_spec(name)[1] + + def triples(stem): + file = path / f'{stem}.txt' + if file.is_file(): + return {tuple(map(int, row.split())) for row in file.read_text().splitlines() if row.strip()} + return {tuple(row) for row in torch.load(path / f'{stem}.pt', weights_only=True).tolist()} + + if data.group == 'transductive': + observed = triples('train') + assert data.candidates == tuple(range(data.context.num_entities)) + elif data.group == 'inductive-e': + observed = triples('train_graph') | triples('val_inference' if split == 'valid' else 'test_inference') + else: + observed = triples('train_graph' if split == 'valid' else 'test_inference') + assert set(data.context.triples) == observed + if data.group != 'transductive': + assert set(data.candidates) == {entity for h, _, t in observed for entity in (h, t)} + scores = torch.sin(torch.arange(data.context.num_entities, dtype=torch.float64)) + report = evaluate_benchmark(data, lambda q: scores, max_queries_per_shape=1) + first = {} + for query in data.queries: + first.setdefault(query.shape, query) + masked = scores.clone() + masked[list(set(range(len(scores))) - set(data.candidates))] = -torch.inf + order = masked.argsort(descending=True).tolist() + positions = {entity: i for i, entity in enumerate(order)} + for shape, query in first.items(): + true_answers = sorted(query.easy | query.hard, key=positions.__getitem__) + ranks = [positions[entity] - offset + 1 for offset, entity in enumerate(true_answers) if entity in query.hard] + expected = dict(mrr=np.mean([1 / r for r in ranks]), + **{f'hits{k}': np.mean([r <= k for r in ranks]) for k in (1, 3, 10)}) + assert {key: report['per_shape'][shape][key] for key in expected} == pytest.approx(expected) + assert report['queries'] == 14 and report['protocol']['all_14_shapes'] + assert not report['protocol']['full_split'] + + +@pytest.mark.integration +@pytest.mark.skipif(not (os.environ.get('DICEE_ULTRAQUERY_DATA_ROOT') and os.environ.get('DICEE_ULTRAQUERY_REFERENCE_ROOT')), + reason='Set official data and pinned upstream roots, with upstream dependencies installed') +@pytest.mark.parametrize('name', ['FB15k237LogicalQuery', 'InductiveFB15k237Query:106', 'WikiTopicsQuery:art']) +@pytest.mark.parametrize('split', ['valid', 'test']) +@pytest.mark.parametrize('tied', [False, True]) +def test_official_ultraquery_evaluator_parity(upstream_reference, name, split, tied): + """Execute the authors' unchanged evaluator on real labels from all 14 types.""" + datasets_query, reference = upstream_reference + + data = load_benchmark(os.environ['DICEE_ULTRAQUERY_DATA_ROOT'], name, split=split) + assert data.metadata['reference_commit'] == REFERENCE_COMMIT + if data.group != 'transductive': + cls = datasets_query.InductiveFB15k237Query if data.group == 'inductive-e' else datasets_query.WikiTopicsQuery + upstream_data = object.__new__(cls) + upstream_data.root = os.environ['DICEE_ULTRAQUERY_DATA_ROOT'] + upstream_data.version = name.split(':')[1] + upstream_data.query_types, upstream_data.union_type = None, 'DNF' + upstream_data.pre_transform = None + if data.group == 'inductive-er': + # DICE's selective extraction omits this unused WikiTopics file; + # the authors' build_vocab ignores validation triples entirely. + upstream_data.load_file = lambda path: ([] if Path(path).stem == 'val_inference' + else cls.load_file(upstream_data, path)) + # Run graph construction without loading training queries or building + # auxiliary relation graphs. This does not modify source data or code. + upstream_data.load_queries = lambda path: None + upstream_data.process() + graph = getattr(upstream_data, f'{split}_graph') + edges = torch.stack((graph.edge_index[0], graph.edge_type, graph.edge_index[1]), dim=1) + assert set(map(tuple, edges.tolist())) == set(data.context.triples) + assert graph.num_nodes == data.context.num_entities + assert graph.num_relations == data.context.num_relations + nodes = getattr(graph, 'restrict_nodes', torch.arange(graph.num_nodes)) + assert tuple(sorted(nodes.tolist())) == data.candidates + grouped = {shape: [] for shape in sorted(ULTRAQUERY_SHAPES)} + for query in data.queries: + grouped[query.shape].append(query) + # Unequal type sizes distinguish macro averaging from query/answer pooling. + selected = tuple(query for i, queries in enumerate(grouped.values()) for query in queries[:1 + i % 3]) + data = replace(data, queries=selected) + n = data.context.num_entities + scores = torch.sin(torch.arange(n, dtype=torch.float64)) + if tied: + scores = scores.round() + report = evaluate_benchmark(data, lambda _: scores) + id2type = [shape + '-DNF' if shape in ('2u', 'up') else shape for shape in grouped] + types = torch.tensor([id2type.index(q.shape + '-DNF' if q.shape in ('2u', 'up') else q.shape) for q in selected]) + easy = torch.zeros((len(selected), n), dtype=torch.bool) + hard = easy.clone() + for i, query in enumerate(selected): + easy[i, sorted(query.easy)] = True + hard[i, sorted(query.hard)] = True + ranks, answer_ranks = reference.batch_evaluate(scores.expand(len(selected), -1).clone(), (types, easy, hard), + torch.tensor(data.candidates)) + expected = reference.evaluate((ranks, torch.zeros(len(selected))), (types, answer_ranks, easy.sum(-1), hard.sum(-1)), + ['mrr', 'hits@1', 'hits@3', 'hits@10'], id2type) + for metric in ('mrr', 'hits1', 'hits3', 'hits10'): + upstream_metric = 'mrr' if metric == 'mrr' else metric.replace('hits', 'hits@') + for shape, upstream_shape in zip(grouped, id2type): + assert report['per_shape'][shape][metric] == pytest.approx(expected[f'[{upstream_shape}] {upstream_metric}']) + for group, label in (('epfo', 'EPFO'), ('negation', 'negation')): + assert report['averages'][group][metric] == pytest.approx(expected[f'[{label}] {upstream_metric}']) + assert report['averages']['all'][metric] == pytest.approx(expected[upstream_metric]) diff --git a/benchmarks/cqa/train.py b/benchmarks/cqa/train.py new file mode 100644 index 000000000..3fc9aa402 --- /dev/null +++ b/benchmarks/cqa/train.py @@ -0,0 +1,296 @@ +"""Train missing UltraQuery comparison weights using clean, pinned author code.""" + +import collections +import collections.abc +import hashlib +import importlib.metadata +import json +import os +import pickle +import platform +import random +import subprocess +import sys +import time +import types +import zlib +from pathlib import Path + +from .manifests import catalog, read_manifest, suite_directory + +PUBLIC = suite_directory('ultraquery') +# Upstream checkout below Experiments/query-baselines/upstream and its pinned commit. +PINS = {'qto': ('qto', catalog().REFERENCES['qto'][1]), + 'inductive-gnnqe': ('inductiveqe', catalog().INDUCTIVE_QE[1])} + + +def sha(path): + with Path(path).open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def write(path, value): + Path(path).write_text(json.dumps(value, indent=2, sort_keys=True) + '\n') + + +def verify_source(source, pin): + if subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() != pin: + raise ValueError('Unexpected training source revision') + if subprocess.check_output(['git', '-C', str(source), 'status', '--porcelain', '--untracked-files=no'], text=True).strip(): + raise ValueError('Training source has modified tracked files') + + +def verify_v2(folder, version): + release = json.loads((PUBLIC / 'inductive-v2-files.json').read_text()) + for filename, expected in release['files'][version].items(): + path = folder / filename + if not path.is_file(): + raise ValueError(f'Missing v2 training input {path}; download the complete official {version}.zip') + crc = 0 + with path.open('rb') as stream: + while block := stream.read(1024 * 1024): + crc = zlib.crc32(block, crc) + if path.stat().st_size != expected['bytes'] or crc != expected['crc32']: + raise ValueError(f'Input differs from corrected v2 release: {path}') + + +def compatibility(): + """Python 3.11 and newer-PyTorch import compatibility; neither GNN arithmetic nor losses change.""" + collections.Sequence = collections.abc.Sequence + drawing = types.ModuleType('rdkit.Chem.Draw.mplCanvas') + drawing.Canvas = object + sys.modules[drawing.__name__] = drawing + from torch.utils import cpp_extension, file_baton + if not hasattr(cpp_extension, 'FileBaton'): # Newer PyTorch no longer re-exports it. + cpp_extension.FileBaton = file_baton.FileBaton + + +def train_qto(job, folder, source, output, args): + import numpy as np + import torch + sys.path.insert(0, str(source / 'kbc/src')) + import datasets + import engines + + # Keep the public integer vocabulary: the authors' text preprocessor + # reindexes first-seen strings, which cannot be used directly by our loader. + name = 'FB15K' + prepared = output / 'kbc-data' + prepared.mkdir() + arrays = {} + for split in ('train', 'valid', 'test'): + arrays[split] = np.array([tuple(map(int, line.split())) for line in (folder / f'{split}.txt').read_text().splitlines() + if line.strip()], dtype=np.int64) + with (prepared / f'{split}.pickle').open('wb') as stream: + pickle.dump(arrays[split], stream) + with (folder / 'id2ent.pkl').open('rb') as stream: + n = len(pickle.load(stream)) + with (folder / 'id2rel.pkl').open('rb') as stream: + nr = len(pickle.load(stream)) + filters = {'lhs': collections.defaultdict(set), 'rhs': collections.defaultdict(set)} + for values in arrays.values(): + for h, r, t in values: + filters['rhs'][int(h), int(r)].add(int(t)) + filters['lhs'][int(t), int(r) + nr].add(int(h)) + with (prepared / 'to_skip.pickle').open('wb') as stream: + pickle.dump({side: {key: sorted(value) for key, value in groups.items()} for side, groups in filters.items()}, stream) + config = dict(dataset=name, device=args.device, reciprocal=False, cache_eval=None, + model_cache_path=str(output) + '/', alias='', seed=str(args.seed), model='ComplEx', + rank=8 if args.smoke else 1000, init=1e-3, regularizer='N3', lmbda=.01, + optimizer='Adagrad', learning_rate=.1, decay1=.9, decay2=.999, dropout=0, + world='LCWA', num_neg=0, score_rel=True, score_rhs=True, score_lhs=False, + w_rel=.1, w_lhs=1., max_epochs=args.epochs or (1 if args.smoke else 100), + batch_size=args.batch_size or (2 if args.smoke else 100), valid=1 if args.smoke else 5) + def setup_ds(options): + dataset = datasets.Dataset(options, data_path=prepared) + dataset.n_entities, dataset.n_predicates = n, nr + return dataset + engines.setup_ds = setup_ds + declared_config = dict(config) + engine = engines.KBCEngine(config) + sampler = engine.dataset.get_sampler('train') + best, best_epoch, updates = -1., None, 0 + for epoch in range(config['max_epochs']): + engine.model.train() + # The upstream while-is_epoch loop consumes the first batch of the + # next epoch before its counter advances. Consume exactly one pass. + for _ in range(0, sampler.size, config['batch_size']): + batch = sampler.batchify(config['batch_size'], args.device) + scores, factors = engine.model(batch, score_rel=True, score_rhs=True, score_lhs=False) + loss = engine.loss(scores[0], batch[:, 2]) + .1 * engine.loss(scores[1], batch[:, 1]) + loss = loss + engine.regularizer.penalty(batch, factors)[0] + if not torch.isfinite(loss): + raise ValueError('Nonfinite QTO training loss') + engine.optimizer.zero_grad() + loss.backward() + engine.optimizer.step() + updates += 1 + if args.smoke: + break + if epoch % config['valid'] == 0: + engine.model.eval() + mrrs, _, _ = engine.dataset.eval(engine.model, 'valid', -1) + score = sum(float(v) for v in mrrs.values()) / len(mrrs) + if score > best: + best, best_epoch = score, epoch + 1 + torch.save(engine.model.state_dict(), output / 'checkpoint.pt') + engine.writer.close() + return dict(configuration=declared_config, model_shape=list(engine.dataset.get_shape()), + best_validation_mrr=best, best_epoch=best_epoch, updates=updates, + adjustments=['Preserve public integer IDs.', + 'Correct the upstream epoch rollover: exactly one pass per epoch.', + 'Select checkpoints on validation only; omit repeated test diagnostics.']) + + +def train_inductive(job, folder, source, output, args): + import torch + compatibility() + sys.path.insert(0, str(source)) + from gnnqe import dataset, gnn, model, task, util # noqa: F401 (imports register TorchDrug classes) + from torchdrug import utils + from torchdrug.layers import functional + if not hasattr(functional, '_size_to_index'): + functional._size_to_index = lambda size: torch.arange(len(size), device=size.device).repeat_interleave(size) + from unittest.mock import patch + + config = util.load_config(source / 'config/complex_query/gnnqe_main.yaml', + context={'ratio': int(job['dataset'].split(':')[1]), 'gpus': 'null' if args.device == 'cpu' else [torch.device(args.device).index or 0]}) + config.engine.batch_size = args.batch_size or (2 if args.smoke else 64) + config.train.num_epoch = args.epochs or (1 if args.smoke else 10) + if args.smoke: + config.train.batch_per_epoch = 1 + config.task.model.model.input_dim = 8 + config.task.model.model.hidden_dims = [8, 8] + config.fast_test = 1 + # Use the authors' local data reader to avoid their unconditional download. + with patch.object(utils, 'download', return_value=str(folder.parent / (folder.name + '.zip'))): + data = dataset.InductiveFB15k237Comp(str(folder.parent), ratio=int(folder.name), verbose=0) + if len(data.split()[1]) < config.fast_test: + config.fast_test = len(data.split()[1]) + solver = util.build_solver(config, data) + config.optimizer.pop('params', None) + # Save the best complete state directly; evaluation validates the graph + # buffers and loads learned tensors into the public inference context. + best, best_epoch, updates = -1., None, 0 + previous = Path.cwd() + os.chdir(output) + try: + for epoch in range(config.train.num_epoch): + solver.model.split = 'train' + solver.train(num_epoch=1, batch_per_epoch=config.train.batch_per_epoch) + updates += config.train.batch_per_epoch + solver.model.split = 'valid' + score = float(solver.evaluate('valid')[config.metric]) + if score > best: + best, best_epoch = score, epoch + 1 + solver.save('checkpoint.pt') + finally: + os.chdir(previous) + return dict(configuration=json.loads(json.dumps(config, default=str)), best_validation_mrr=best, + best_epoch=best_epoch, updates=updates, + adjustments=['Use local corrected v2 inputs; omit final test inference during training.', + 'Configure TorchDrug size-to-index helper and Python/RDKit imports.']) + + +def add_arguments(parser): + parser.add_argument('--methods', nargs='+', choices=tuple(PINS), required=True) + parser.add_argument('--datasets', nargs='+') + parser.add_argument('--data-root', default='KGs/UltraQuery') + parser.add_argument('--output', type=Path, required=True, help='Training directory, below the input root for Docker') + parser.add_argument('--manifest-output-root', type=Path, help='Path of OUTPUT as later evaluations see it, below the input root') + parser.add_argument('--device', default='cpu') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--threads', type=int, default=2) + parser.add_argument('--batch-size', type=int) + parser.add_argument('--epochs', type=int) + parser.add_argument('--reference-deps', type=Path, nargs='+', default=[], help='Extra import roots for author dependencies') + parser.add_argument('--smoke', action='store_true', help='Tiny model and one-step fixture test; never paper weights') + + +def run(args): + """Train the selected recipes of ``trained_baselines.json``, each author codebase in its own process.""" + if args.threads < 1 or any(v is not None and v < 1 for v in (args.batch_size, args.epochs)): + raise ValueError('Positive threads, batch size and epoch count required') + recipes = read_manifest(PUBLIC / 'trained_baselines.json') + jobs = [e for e in recipes['entries'] if e['method'] in args.methods + and (args.datasets is None or e['dataset'] in args.datasets)] + if not jobs or (args.datasets and set(args.datasets) - {e['dataset'] for e in jobs}): + raise ValueError('Select compatible methods/datasets from trained_baselines.json') + if args.dry_run: + print(json.dumps(dict(device=args.device, jobs=[dict(id=j['id'], dataset=j['dataset'], method=j['method']) for j in jobs], + seed=args.seed, smoke=args.smoke), indent=2)) + return + if args.device == 'cpu': + os.environ['CUDA_VISIBLE_DEVICES'] = '' + for path in reversed(args.reference_deps): + sys.path.insert(0, str(path.resolve())) + if (path / 'bin').is_dir(): + os.environ['PATH'] = str(path.resolve() / 'bin') + os.pathsep + os.environ['PATH'] + import numpy as np + import torch + torch.set_num_threads(args.threads) + torch.set_float32_matmul_precision('highest') + args.input_root, args.output = args.input_root.resolve(), args.output.resolve() + manifest_root = (args.manifest_output_root or args.output).resolve() + if args.manifest_output_root and not manifest_root.is_relative_to(args.input_root): + raise ValueError('Manifest output root must be below input-root') + for job in jobs: + # Separate interpreters prevent registry/name collisions between authors. + if len(jobs) > 1: + command = [sys.executable, '-m', 'benchmarks.cqa', 'ultraquery', 'train', '--methods', job['method'], + '--datasets', job['dataset'], '--input-root', str(args.input_root), '--data-root', args.data_root, + '--output', str(args.output), '--device', args.device, '--threads', str(args.threads), '--seed', str(args.seed)] + for option, value in [('--batch-size', args.batch_size), ('--epochs', args.epochs)]: + if value is not None: + command += [option, str(value)] + if args.manifest_output_root: + command += ['--manifest-output-root', str(manifest_root)] + if args.reference_deps: + command += ['--reference-deps', *map(str, args.reference_deps)] + if args.smoke: + command.append('--smoke') + subprocess.run(command, check=True) + continue + checkout, pin = PINS[job['method']] + source = args.input_root / 'Experiments/query-baselines/upstream' / checkout + verify_source(source, pin) + folder = args.input_root / args.data_root / ('FB15k-betae' if job['method'] == 'qto' else job['dataset'].split(':')[1]) + if job['method'] == 'inductive-gnnqe' and not args.smoke: + verify_v2(folder, job['dataset'].split(':')[1]) + output = args.output / job['id'] + if output.exists(): + raise FileExistsError(f'Use a fresh training output: {output}') + output.mkdir(parents=True) + provenance = dict(version=1, method=job['method'], dataset=job['dataset'], source_commit=pin, + launcher_sha256=sha(__file__), seed=args.seed, device=args.device, threads=args.threads, + profile='smoke' if args.smoke else 'custom' if args.epochs or args.batch_size else 'author-recipe', + dataset_release='fixture' if args.smoke else 'InductiveQE v2.0' if job['method']=='inductive-gnnqe' else 'BetaE', + inputs={p.name: sha(p) for p in sorted(folder.iterdir()) if p.is_file()}, + environment=dict(python=platform.python_version(), torch=str(torch.__version__), + image_id=os.environ.get('DICEE_IMAGE_ID'), + packages={d.metadata['Name']: d.version for d in importlib.metadata.distributions() + if d.metadata['Name']}), + checkpoint_selection='validation MRR only', status='running') + write(output / 'training.json', provenance) + torch.manual_seed(args.seed) + random.seed(args.seed) + np.random.seed(args.seed) + started = time.monotonic() + try: + result = (train_qto if job['method']=='qto' else train_inductive)(job, folder, source, output, args) + provenance.update(result, status='complete', seconds=time.monotonic()-started, + checkpoint_sha256=sha(output / 'checkpoint.pt')) + write(output / 'training.json', provenance) + logical = manifest_root / job['id'] + def manifest_path(filename): + path = logical / filename + return str(path.relative_to(args.input_root) if path.is_relative_to(args.input_root) else path) + local = dict(recipes, seed=args.seed, threads=args.threads, data_root=args.data_root, + entries=[dict(job, checkpoint=manifest_path('checkpoint.pt'), training=manifest_path('training.json'))]) + local['entries'][0]['reference']['training_profile'] = provenance['profile'] + local['entries'][0]['blockers'] = ['Training smoke checkpoint; not a paper baseline'] if args.smoke else [] + write(output / 'manifest.json', local) + except Exception as error: + provenance.update(status='failed', error=str(error)) + write(output / 'training.json', provenance) + raise diff --git a/benchmarks/cqa/training/Dockerfile b/benchmarks/cqa/training/Dockerfile new file mode 100644 index 000000000..e737bcd33 --- /dev/null +++ b/benchmarks/cqa/training/Dockerfile @@ -0,0 +1,17 @@ +# CPU author runtime. Build from an immutable shared benchmark image ID. +ARG BENCHMARK_IMAGE +FROM ${BENCHMARK_IMAGE} +USER root +RUN apt-get -o Acquire::Retries=5 update \ + && apt-get install -y --no-install-recommends git libxrender1 libxext6 libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* +COPY requirements.lock /tmp/author-requirements.lock +# TorchDrug's Python upper bound predates 3.11; the launcher supplies import +# compatibility. Keep base dependencies intact and verify every added wheel. +RUN pip install --no-cache-dir --no-deps --require-hashes --ignore-requires-python \ + -r /tmp/author-requirements.lock +COPY torchdrug_compat.py /opt/dicee/benchmarks/cqa/training/torchdrug_compat.py +RUN python /opt/dicee/benchmarks/cqa/training/torchdrug_compat.py > /opt/dicee/benchmarks/cqa/training/torchdrug-compat.json +ENV MAX_JOBS=2 TORCH_EXTENSIONS_DIR=/cache/torch_extensions MPLCONFIGDIR=/cache/matplotlib +# TorchDrug also creates short-lived compiler locks in the working directory. +WORKDIR /tmp diff --git a/benchmarks/cqa/training/requirements.lock b/benchmarks/cqa/training/requirements.lock new file mode 100644 index 000000000..43585eb8b --- /dev/null +++ b/benchmarks/cqa/training/requirements.lock @@ -0,0 +1,22 @@ +# Author runtime additions to the locked +H/UltraQuery image; Python 3.11 / Linux x86_64. +# Install without dependencies: the base image supplies the remaining pinned dependencies. +torchdrug==0.2.1 --hash=sha256:f416dbbfb3fc697770e9ee7fa7769611fad060a4749ee49499d87583cfb58ee2 +torch-geometric==2.6.1 --hash=sha256:8faeb353f9655f7dbec44c5e0b44c721773bdfb279994da96b9b8b12fd30f427 +ninja==1.13.2 --hash=sha256:65a24341b5ac09fcadcc37082660be40a94174e51a937fabf6e2cae26225fa2c +fair-esm==2.0.0 --hash=sha256:7dcb58828a401155d1457d071a03800bd1a50331ddf24ac79ad1dd0f2ea682a2 +easydict==1.13 --hash=sha256:6b787daf4dcaf6377b4ad9403a5cee5a86adbc0ca9a5bcf5410e9902002aeac2 +decorator==5.3.1 --hash=sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c +lmdb==2.3.0 --hash=sha256:4d0019870b5b93b2160fc324a609ffee1447024b2b8ce0da3b3f2f04b0131870 +rdkit==2026.3.6 --hash=sha256:dcf09347e1a0dc1d7e5fcc34f5da79d1f4bb0e04ecadee669449b82ae2492799 +pillow==12.3.0 --hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd +matplotlib==3.11.2 --hash=sha256:07d9b9fa60cd4c393692f50d0bb03123242ddf61c99bb0e95e75feb354e7c1a8 +pyparsing==3.3.3 --hash=sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4 +cycler==0.12.1 --hash=sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 +kiwisolver==1.5.1 --hash=sha256:95a02752aa032eef4aed01cda6d9b687c669bd0396bf4519eef8bba22a286720 +fonttools==4.66.1 --hash=sha256:72299346b96b9244dabcc051b24e4653da4edfda6105544cfb10ce856a1afaac +contourpy==1.3.3 --hash=sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db +ogb==1.3.6 --hash=sha256:29ab84078c66a7846bf137c9be9545978616a053e734c032a2ff731d026bb5e9 +outdated==0.2.2 --hash=sha256:3e9c2ee6d17e86ae8cc7bb71d70c4172690121cda367155a30994742172678c8 +littleutils==0.2.4 --hash=sha256:d10d5fe2e107c49fe2fc2904a08d6e5a302b41f8405921835ffcc323782d5dbc +torch_scatter @ https://data.pyg.org/whl/torch-2.9.0%2Bcu128/torch_scatter-2.1.2%2Bpt29cu128-cp311-cp311-linux_x86_64.whl --hash=sha256:58b82728cd48fe565ceae3a049b58f8a1378dbdcae75d24aecdf870a29196331 +torch_cluster @ https://data.pyg.org/whl/torch-2.9.0%2Bcu128/torch_cluster-1.6.3%2Bpt29cu128-cp311-cp311-linux_x86_64.whl --hash=sha256:f605601a14406d65c904297437b1e56cb339cb7db8ba48a39397a31a1703eb51 diff --git a/benchmarks/cqa/training/torchdrug_compat.py b/benchmarks/cqa/training/torchdrug_compat.py new file mode 100644 index 000000000..6ed453b3d --- /dev/null +++ b/benchmarks/cqa/training/torchdrug_compat.py @@ -0,0 +1,49 @@ +"""Apply three mechanical TorchDrug 0.2.1 fixes for the pinned PyTorch runtime. + +No kernel arithmetic changes: update two moved includes and pass loader options +by keyword because the modern PyTorch positional signature has changed. +""" + +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path + + +def apply(root): + replacements = { + 'layers/functional/extension/spmm.h': ('#include ', + '#include '), + 'layers/functional/extension/rspmm.h': ('#include ', + '#include '), + 'utils/torch.py': ('self.extra_ldflags, self.extra_include_paths, self.build_directory,\n' + ' self.verbose, **self.kwargs)', + 'extra_ldflags=self.extra_ldflags, extra_include_paths=self.extra_include_paths,\n' + ' build_directory=self.build_directory, verbose=self.verbose, **self.kwargs)'), + } + pending = [] + for relative, (old, new) in replacements.items(): + path = root / relative + original = path.read_text() + if old not in original and new not in original: + raise ValueError(f'Unexpected TorchDrug source; refusing to patch {path}') + pending.append((path, original, original.replace(old, new))) + report = {} + for path, original, updated in pending: + if original != updated: + path.write_text(updated) + report[str(path.relative_to(root))] = dict( + before_sha256=hashlib.sha256(original.encode()).hexdigest(), + after_sha256=hashlib.sha256(updated.encode()).hexdigest()) + return report + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('root', nargs='?', type=Path) + args = parser.parse_args() + root = args.root + if root is None: + root = Path(importlib.util.find_spec('torchdrug').origin).parent + print(json.dumps(apply(root), indent=2)) diff --git a/benchmarks/cqa/verification/__init__.py b/benchmarks/cqa/verification/__init__.py new file mode 100644 index 000000000..d246a948c --- /dev/null +++ b/benchmarks/cqa/verification/__init__.py @@ -0,0 +1 @@ +"""Independent reference implementations and parity checks.""" diff --git a/benchmarks/cqa/verification/export_reference.py b/benchmarks/cqa/verification/export_reference.py new file mode 100644 index 000000000..9361787e1 --- /dev/null +++ b/benchmarks/cqa/verification/export_reference.py @@ -0,0 +1,509 @@ +"""Export full-candidate validation oracles from upstream code, without DICE imports.""" + +import argparse +import collections +import collections.abc +import hashlib +import importlib.util +import json +import pickle +import platform +import random +import re +import subprocess +import sys +import types +from pathlib import Path +from types import FunctionType + +import numpy as np +import torch +from torch.overrides import TorchFunctionMode + + +class ReferenceAttention(TorchFunctionMode): + """Partition independent heads; retain each complete reference query batch.""" + def __torch_function__(self, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + if func is torch.nn.functional.multi_head_attention_forward: + namespace = func.__globals__ | {'scaled_dot_product_attention': lambda *a, **kw: self.__torch_function__( + torch.nn.functional.scaled_dot_product_attention, (), a, kw)} + copied = FunctionType(func.__code__, namespace, func.__name__, func.__defaults__, func.__closure__) + return copied(*args, **kwargs) + if func is torch.nn.functional.scaled_dot_product_attention: + q, k, v = args[:3] + if q.is_cuda and q.shape[-3] > 1 and q.shape[-2] * k.shape[-2] > 1024**2: + mask = args[3] if len(args) > 3 else kwargs.get('attn_mask') + dropout = args[4] if len(args) > 4 else kwargs.get('dropout_p', 0) + if mask is not None or dropout or kwargs.get('enable_gqa', False): + raise ValueError('Reference attention expects unmasked inference') + heads = [func(q.narrow(-3, i, 1), k.narrow(-3, i, 1), v.narrow(-3, i, 1), *args[3:], **kwargs) + for i in range(q.shape[-3])] + return torch.cat(heads, dim=-3) + return func(*args, **kwargs) + + +# Query types an upstream oracle cannot cover; benchmarks/cqa/oracles.py accepts exactly these exemptions. +NEGATED_TYPES = ('2in', '3in', 'inp', 'pin', 'pni') +EXEMPTION_REASON = ('No upstream reference: the pinned +H CQD has no negation in its beam search. The negated types use CQD-A ' + 'signed-atom negation, checked against CQD-A outputs in ' + 'tests/test_query_baselines.py::test_opt_in_cqd_negation_matches_cqda.') + + +def exempt_types(entry): + """The pinned +H CQD has no negation in its beam search, so CQD-A negated types have no upstream reference.""" + if entry['method'] in ('cqd', 'cqd-hybrid') and entry['options'].get('atomic_negation'): + return sorted(set(entry['query_types']) & set(NEGATED_TYPES)) + return [] + + +FORMULAS = { + '1p': 'r1(s1,f)', '2p': '(r1(s1,e1))&(r2(e1,f))', + '3p': '((r1(s1,e1))&(r2(e1,e2)))&(r3(e2,f))', + '4p': '(((r1(s1,e1))&(r2(e1,e2)))&(r3(e2,e3)))&(r4(e3,f))', + '2i': '(r1(s1,f))&(r2(s2,f))', '3i': '((r1(s1,f))&(r2(s2,f)))&(r3(s3,f))', + '4i': '(((r1(s1,f))&(r2(s2,f)))&(r3(s3,f)))&(r4(s4,f))', + 'ip': '((r1(s1,e1))&(r2(s2,e1)))&(r3(e1,f))', + 'pi': '((r1(s1,e1))&(r2(e1,f)))&(r3(s2,f))', + '2in': '(r1(s1,f))&(!(r2(s2,f)))', '3in': '((r1(s1,f))&(r2(s2,f)))&(!(r3(s3,f)))', + 'inp': '((r1(s1,e1))&(!(r2(s2,e1))))&(r3(e1,f))', + 'pin': '((r1(s1,e1))&(r2(e1,f)))&(!(r3(s2,f)))', + 'pni': '((r1(s1,e1))&(!(r2(e1,f))))&(r3(s2,f))', +} + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def flatten(value): + return [i for part in value for i in flatten(part)] if isinstance(value, tuple) else [value] + + +def structure(query): + if isinstance(query[0], int): + return 'e', tuple('n' if r == -2 else 'r' for r in query[1]) + if len(query) == 2 and all(isinstance(r, int) for r in query[1]): + return structure(query[0]), tuple('n' if r == -2 else 'r' for r in query[1]) + return tuple(('u',) if part == (-1,) else structure(part) for part in query) + + +def module(path, name): + spec = importlib.util.spec_from_file_location(name, path) + result = importlib.util.module_from_spec(spec) + spec.loader.exec_module(result) + return result + + +# The pinned upstream revisions, read from the import-free catalog (no DICE imports). +COMMITS = {method: commit for method, (_, commit) in + module(Path(__file__).resolve().parents[3] / 'dicee/query_answering/catalog.py', 'cqa_catalog').REFERENCES.items()} + + +def load_state(path): + state = torch.load(path, map_location='cpu', weights_only=True) + while any(key in state for key in ('model', 'model_state_dict', 'state_dict')): + state = next(state[key] for key in ('model', 'model_state_dict', 'state_dict') if key in state) + return state + + +def reference_dataset(bundle, entry, root): + """Read published layouts independently; never import DICE's dataset loader.""" + name = entry['dataset'] + plus_h = {'FB15k237+H': 'FB15k-237+H', 'NELL995+H': 'NELL995+H', 'ICEWS18+H': 'ICEWS18+H'} + transductive = {'FB15k237LogicalQuery': 'FB15k-237-betae', 'FB15kLogicalQuery': 'FB15k-betae', + 'NELL995LogicalQuery': 'NELL-betae'} + if name in plus_h: + folder = Path('iscqa-compl-benchmarks/new_benchmarks') / plus_h[name] + elif name in transductive: + folder = Path(transductive[name]) + else: + family, version = name.split(':') + if family not in ('InductiveFB15k237Query', 'WikiTopicsQuery'): + raise ValueError('Unsupported reference dataset') + folder = Path(version) if family == 'InductiveFB15k237Query' else Path('WikiTopics_QE') / version + prefix = str(Path(bundle['manifest']['data_root']) / folder) + '/' + raw = Path(root) / prefix + + def unpickle(filename): + with (raw / filename).open('rb') as stream: + return pickle.load(stream) + + def triples(stem): + path = raw / (stem + '.txt') + if path.is_file(): + return [tuple(map(int, line.split())) for line in path.read_text().splitlines() if line.strip()] + return list(map(tuple, torch.load(raw / (stem + '.pt'), map_location='cpu', weights_only=True).tolist())) + + def context(edges, n, nr, pairs): + inverse = {a: b for h, t in pairs for a, b in ((h, t), (t, h))} + edges = sorted(set(edges) | {(t, inverse[r], h) for h, r, t in edges if r in inverse}) + return dict(triples=edges, num_entities=n, num_relations=nr, inverse_relations=sorted(pairs)) + + if name in plus_h or name in transductive: + n, nr = len(unpickle('id2ent.pkl')), len(unpickle('id2rel.pkl')) + stem = 'KG_splits/train' if name == 'ICEWS18+H' else 'train' + train = triples(stem) + test = train + if name in plus_h and (entry.get('inference_graph') or 'train+valid') == 'train+valid': + test = train + triples('KG_splits/valid' if name == 'ICEWS18+H' else 'valid') + pairs = [(r, r + 1) for r in range(0, nr, 2)] + graphs = {'valid': context(train, n, nr, pairs), 'test': context(test, n, nr, pairs)} + structured, easy, hard = [unpickle('valid-' + kind + '.pkl') for kind in ('queries', 'easy-answers', 'hard-answers')] + else: + train, test = triples('train_graph'), triples('test_inference') + if family == 'InductiveFB15k237Query': + valid = triples('val_inference') + vocabulary = train + valid + test + n = max(entity for h, _, t in vocabulary for entity in (h, t)) + 1 + nr = max(r for _, r, _ in vocabulary) + 1 + pairs = [(r, r + nr // 2) for r in range(nr // 2)] + graphs = {split: context(train + edges, n, nr, pairs) for split, edges in [('valid', valid), ('test', test)]} + else: + mapping = unpickle('og_mappings.pkl')['r2id'] if (raw / 'og_mappings.pkl').is_file() else None + graphs = {} + for split, edges in [('valid', train), ('test', test)]: + n = max(entity for h, _, t in edges for entity in (h, t)) + 1 + nr = max(r for _, r, _ in edges) + 1 + pairs = ([(r, mapping[label + '_inv']) for label, r in mapping.items() + if label + '_inv' in mapping and r < nr and mapping[label + '_inv'] < nr] + if mapping else [(r, r + nr // 2) for r in range(nr // 2)]) + graphs[split] = context(edges, n, nr, pairs) + structured, grouped_easy, grouped_hard = [unpickle('valid_' + kind + '.pkl') + for kind in ('queries', 'answers_easy', 'answers_hard')] + easy = {query: answers for group in grouped_easy.values() for query, answers in group.items()} + hard = {query: answers for group in grouped_hard.values() for query, answers in group.items()} + graph = graphs['valid'] + return prefix, structured, easy, hard, graph, {split: digest(value) for split, value in graphs.items()} + + +def predictor(entry, upstream, checkpoint, n, nr, triples, queries, device, probe_queries=None): + method, options = entry['method'], entry['options'] + names = {structure(query): shape + '-DNF' if shape in ('2u', 'up') else shape for shape, query in queries.values()} + if method == 'cone': + source, state = module(upstream / 'models.py', 'reference_cone'), load_state(checkpoint) + dim = state['entity_embedding'].shape[1] + model = source.KGReasoning(n, nr, dim, state['gamma'].item(), use_cuda=device.startswith('cuda'), + query_name_dict=names, center_reg=options['center_reg']) + model.cone_proj = source.ConeProjection(dim, state['cone_proj.layer1.weight'].shape[0], 2) + model.load_state_dict(state, strict=True) + model.to(device).eval() + + def predict(shape, batch): + encoded = {structure(batch[0]): torch.tensor([flatten(q) for q in batch], device=device)} + indices = {structure(batch[0]): list(range(len(batch)))} + return torch.cat([model(None, torch.arange(start, min(start + 1024, n), device=device)[None] + .expand(len(batch), -1), None, encoded, indices)[1] + for start in range(0, n, 1024)], dim=-1) + return predict + if method in ('cqd', 'cqd-hybrid'): + from cqd.base import CQD + state = load_state(checkpoint) + filters = collections.defaultdict(list) + for h, r, t in triples: + filters[h, r].append(t) + model = CQD(n, nr, state['embeddings.0.weight'].shape[1] // 2, k=options['beam_size'], max_k=options['max_k'], + max_norm=.9 if method == 'cqd-hybrid' else 1., filters=filters, + do_normalize=options['normalize'], t_norm_name=options['tnorm'], query_name_dict=names) + model.load_state_dict(state, strict=True) + model.to(device).eval() + + def predict(shape, batch): + settings = options | options['per_shape'].get(shape, {}) + model.k, model.t_norm_name, model.max_k = settings['beam_size'], settings['tnorm'], settings['max_k'] + with torch.device(device): + if settings.get('reference_batching') is False: + from cqd import discrete + from reference_cqd import query_bounded + return query_bounded(model, discrete, shape, torch.tensor([flatten(q) for q in batch], device=device), + settings['row_batch_size']) + if shape == '4p' and options.get('final_batch_size') and device.startswith('cuda'): + from cqd import discrete + from reference_cqd import query_4p + return query_4p(model, discrete, torch.tensor([flatten(q) for q in batch], device=device), + options['final_batch_size']) + return model(None, None, None, {structure(batch[0]): torch.tensor([flatten(q) for q in batch], device=device)}, + {structure(batch[0]): list(range(len(batch)))})[1] + return predict + if method == 'clmpt': + from src.language.foq import EFO1Query + from src.language.grammar import parse_lstr_to_lformula + from src.pipeline.clmpt import CLMPTLayer, CLMPTReasoner + from src.structure.nbp_complex import ComplEx + state = load_state(checkpoint) + pre_norm = any(key.startswith('transformer.encoder.') for key in state) + backbone = ComplEx(n, nr, state['nbp._entity_embedding.weight'].shape[1] // 2, device=device) + hidden = state['transformer.encoder.layers.0.ffn.layer1.weight' if pre_norm else 'transformer.layers.0.linear1.weight'].shape[0] + model = CLMPTLayer(hidden, backbone, layers=2, pre_norm=pre_norm, agg_func=options['aggregation']) + model.load_state_dict(state, strict=True) + model.to(device).eval() + reasoner = CLMPTReasoner(backbone, model) + + def atomic(shape, batch): + formula = FORMULAS[shape] + symbols = [] + for relation, source, _ in re.findall(r'(r\d+)\((\w+),(\w+)\)', formula): + for key in (source, relation): + if key.startswith(('s', 'r')) and key not in symbols: + symbols.append(key) + query = EFO1Query(parse_lstr_to_lformula(formula)) + for row in batch: + query.append_qa_instances(dict(zip(symbols, [value for value in flatten(row) if value >= 0])), {'f': []}, {'f': []}) + reasoner.initialize_with_query(query) + with ReferenceAttention(): + reasoner.estimate_variable_embeddings() + embeddings = reasoner.get_ent_emb('f')[:probe_queries] + return torch.stack([torch.cat([torch.cosine_similarity(row[None], entities, dim=-1) + for entities in backbone.entity_embedding.split(options['candidate_batch_size'])]) for row in embeddings]) + + def predict(shape, batch): + if shape == '2u': + return torch.maximum(atomic('1p', [q[0] for q in batch]), atomic('1p', [q[1] for q in batch])) + if shape == 'up': + return torch.maximum(*[atomic('2p', [(q[0][branch][0], (*q[0][branch][1], *q[1])) for q in batch]) for branch in (0, 1)]) + return atomic(shape, batch) + return predict + if method in ('ultraquery', 'ultraquery-lp'): + from torch_geometric.data import Data + from ultra import datasets_query, tasks # noqa: F401 + from ultra.models import Ultra + from ultra.query_utils import Query + from ultra.ultraquery import UltraQuery + edges = torch.tensor(triples) + graph = Data(edge_index=edges[:, [0, 2]].T, edge_type=edges[:, 1], num_nodes=n, num_relations=nr) + tasks.build_relation_graph(graph) + graph = graph.to(device) + state = load_state(checkpoint) + sentinel = 'relation_model.layers.0.linear.weight' + prefix = next(p for p in ('model.model.', 'model.', '') if p + sentinel in state) + state = {k[len(prefix):]: v for k, v in state.items()} + key = sentinel + dim = state[key].shape[0] + layers = len({k.split('.')[2] for k in state if k.startswith('relation_model.layers.')}) + config = dict(input_dim=dim, hidden_dims=[dim] * layers, message_func='distmult', aggregate_func='sum', short_cut=True, layer_norm=True) + backbone = Ultra(dict(config, **{'class': 'RelNBFNet'}), dict(config, **{'class': 'QueryNBFNet'})) + model = UltraQuery(backbone, threshold=options['threshold'], logic=options['logic']) + backbone.load_state_dict(state, strict=True) + model.to(device).eval() + if options.get('calibration') == 'softmax-degree': + # DICE's opt-in calibrated variant, built from upstream parts: upstream's relation and entity models give the + # projection logits; the softmax over entities times the observed-tail mass that upstream's SymbolicTraversal + # reaches from the fuzzy set (at least one), capped at 0.9999, replaces the sigmoid; with observed_traversal + # the result also takes the maximum with that traversal. + from ultra.ultraquery import SymbolicTraversal + if options['threshold']: + raise ValueError('A calibrated projection takes no membership threshold') + projection, symbolic, traverse = model.model, SymbolicTraversal(), bool(options.get('observed_traversal')) + + def calibrated(graph, h_prob, r_index): + rel_reprs = projection.model.relation_model(graph.relation_graph, query=r_index) + query = rel_reprs[torch.arange(r_index.shape[0], device=r_index.device), r_index] + output = projection.model.entity_model(graph, torch.einsum('bn, bd -> bnd', h_prob, query), rel_reprs, query) + known = symbolic(graph, h_prob, r_index) + result = (output.log_softmax(1) + known.sum(1, keepdim=True).clamp(min=1).log()).exp().clamp(max=.9999) + return torch.maximum(result, known) if traverse else result + model.model.forward = calibrated + elif options.get('observed_traversal'): + # DICE's opt-in variant, built from upstream parts: each projection also takes the maximum with upstream's + # own symbolic traversal of the same fuzzy set, so observed facts keep their source's membership. + from ultra.ultraquery import SymbolicTraversal + neural, symbolic = model.model.forward, SymbolicTraversal() + model.model.forward = lambda graph, h_prob, r_index: torch.maximum(neural(graph, h_prob, r_index), + symbolic(graph, h_prob, r_index)) + return lambda shape, batch: model(graph, torch.stack([Query.from_nested(q) for q in batch]).to(device), symbolic_traversal=False) + if method in ('gnnqe', 'inductive-gnnqe', 'incoming-relation'): + collections.Sequence = collections.abc.Sequence + drawing = types.ModuleType('rdkit.Chem.Draw.mplCanvas') + drawing.Canvas = object + sys.modules[drawing.__name__] = drawing + from gnnqe.data import Query + from gnnqe.gnn import NeuralBellmanFordNetwork + from gnnqe.model import QueryExecutor + from torchdrug import data + from torchdrug.layers import functional + if not hasattr(functional, '_size_to_index'): + functional._size_to_index = lambda size: torch.arange(len(size), device=size.device).repeat_interleave(size) + graph = data.Graph(torch.tensor(triples)[:, [0, 2, 1]], num_node=n, num_relation=nr).to(device) + if method == 'incoming-relation': + from gnnqe.heuristic_baseline import HeuristicBaseline + model = HeuristicBaseline().to(device).eval() + seed = options.get('seed', torch.initial_seed()) + # Building TorchDrug's random perfect-hash index must not consume + # the query's random ranking stream. + graph.match(torch.tensor([[-1, -1, r] for r in range(nr)], device=device)) + + def heuristic_predict(shape, batch): + scores = [] + devices = [torch.device(device).index or 0] if device.startswith('cuda') else [] + for q in batch: + payload = json.dumps([seed, flatten(q)], separators=(',', ':')).encode() + local_seed = int.from_bytes(hashlib.sha256(payload).digest()[:8], 'little') % (2**63) + with torch.random.fork_rng(devices=devices): + torch.manual_seed(local_seed) + scores.append(model(graph, Query.from_nested(q)[None].to(device))[0]) + return torch.stack(scores) + return heuristic_predict + with torch.serialization.safe_globals([data.Graph]): + state = torch.load(checkpoint, map_location='cpu', weights_only=True)['model'] + buffers = ('train_graph', 'valid_graph', 'test_graph', 'valid_nodes', 'test_nodes') if method == 'inductive-gnnqe' else ('fact_graph', 'graph') + state = {key.removeprefix('model.'): value for key, value in state.items() if key not in buffers} + dim = state['model.query.weight'].shape[1] + count = len({key.split('.')[3] for key in state if key.startswith('model.model.layers.')}) + backbone = NeuralBellmanFordNetwork(dim, [dim] * count, nr, aggregate_func='pna', short_cut=True, layer_norm=True, + concat_hidden=state['model.mlp.layers.0.weight'].shape[1] != 2 * dim) + model = QueryExecutor(backbone, logic=options['logic']) + model.load_state_dict(state, strict=True) + model.to(device).eval() + return lambda shape, batch: model(graph, torch.stack([Query.from_nested(q) for q in batch]).to(device)) + if method == 'qto': + source = module(upstream / 'model.py', 'reference_qto') + from reference_qto import ReferenceRelations + from src.models import ComplEx + state = load_state(checkpoint) + backbone = ComplEx([n, nr, n], rank=state['embeddings.0.weight'].shape[1] // 2, init_size=1.) + backbone.load_state_dict(state, strict=True) + backbone.to(device).eval() + + model = source.KGReasoning.__new__(source.KGReasoning) + torch.nn.Module.__init__(model) + model.nentity, model.device, model.fraction, model.neg_scale = n, device, min(n, 100), options['negation_scale'] + model.relation_embeddings = ReferenceRelations(source, backbone, triples, n, device, options['threshold'], model.fraction) + return lambda shape, batch: model.embed_query(torch.tensor([flatten(q) for q in batch], device=device), structure(batch[0]), 0)[0] + raise ValueError('Only upstream baselines have an independent reference exporter') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--bundle', type=Path, required=True) + parser.add_argument('--input-root', type=Path, required=True) + parser.add_argument('--entry', required=True) + parser.add_argument('--upstream', type=Path, required=True) + parser.add_argument('--device', default='cpu') + parser.add_argument('--pilot-queries', type=int, default=2) + parser.add_argument('--probe-only', action='store_true', help='Score only the pilot queries after encoding complete reference batches') + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if args.pilot_queries < 1: + parser.error('Positive pilot budget required') + bundle = json.loads((args.bundle / 'bundle.json').read_text()) + if sha(args.bundle / 'bundle.json') != json.loads((args.bundle / 'bundle.sha256.json').read_text())['sha256']: + raise ValueError('Bundle checksum mismatch') + entries = {entry['id']: entry for entry in bundle['manifest']['entries']} + if args.entry not in entries: + parser.error(f'No frozen entry {args.entry!r} in this bundle') + entry = entries[args.entry] + commit = subprocess.check_output(['git', '-C', str(args.upstream), 'rev-parse', 'HEAD'], text=True).strip() + if commit != COMMITS.get(entry['method']): + raise ValueError('Unexpected upstream revision') + if subprocess.check_output(['git', '-C', str(args.upstream), 'status', '--porcelain', '--untracked-files=no'], text=True).strip(): + raise ValueError('Upstream tracked files differ from the pinned revision') + for relative, expected in bundle['files'].items(): + if sha(args.input_root / relative) != expected: + raise ValueError(f'Input checksum mismatch: {relative}') + torch.set_num_threads(bundle['manifest']['threads']) + seed = bundle['manifest']['seed'] + torch.manual_seed(seed) + random.seed(seed) + np.random.seed(seed) + torch.backends.cuda.matmul.fp32_precision = 'ieee' + torch.backends.cudnn.conv.fp32_precision = 'ieee' + sys.path.insert(0, str(args.upstream.resolve())) + if entry['method'] == 'qto': + sys.path.insert(0, str(args.upstream.resolve() / 'kbc')) + prefix, structured, easy, hard, graph, graphs = reference_dataset(bundle, entry, args.input_root) + n, nr, triples = graph['num_entities'], graph['num_relations'], graph['triples'] + name_shapes = {} + for shape in entry['query_types']: + # Infer structure labels independently from the source query syntax. + example = {'1p': (0, (0,)), '2p': (0, (0, 0)), '3p': (0, (0, 0, 0)), '4p': (0, (0, 0, 0, 0)), + '2i': ((0, (0,)), (1, (0,))), '3i': ((0, (0,)), (1, (0,)), (2, (0,))), + '4i': ((0, (0,)), (1, (0,)), (2, (0,)), (3, (0,))), + 'ip': (((0, (0,)), (1, (0,))), (0,)), 'pi': ((0, (0, 0)), (1, (0,))), + '2u': ((0, (0,)), (1, (0,)), (-1,)), 'up': (((0, (0,)), (1, (0,)), (-1,)), (0,)), + '2in': ((0, (0,)), (1, (0, -2))), '3in': ((0, (0,)), (1, (0,)), (2, (0, -2))), + 'inp': (((0, (0,)), (1, (0, -2))), (0,)), 'pin': ((0, (0, 0)), (1, (0, -2))), + 'pni': ((0, (0, 0, -2)), (1, (0,)))}[shape] + name_shapes[structure(example)] = shape + queries = {digest((name_shapes[struct], query, sorted(easy[query]), sorted(hard[query]))): (name_shapes[struct], query) + for struct, group in structured.items() if struct in name_shapes for query in group} + frozen_plan = bundle['plans'][f'{args.entry}/valid'] + if sha(args.bundle / frozen_plan['path']) != frozen_plan['sha256']: + raise ValueError('Plan checksum mismatch') + plan = json.loads((args.bundle / frozen_plan['path']).read_text()) + selected_inputs = {entry['checkpoint'], *entry.get('adapters', {}).values()} + if entry.get('training'): + selected_inputs.add(entry['training']) + inputs = {key: value for key, value in bundle['files'].items() if key in selected_inputs or key.startswith(prefix)} + oracle = dict(version=1, entry_sha256=digest({key: value for key, value in entry.items() if key != 'verification'}), + inputs_sha256=digest(inputs), validation_plan_sha256=frozen_plan['sha256'], + reference_commit=commit, exporter_sha256=sha(Path(__file__)), pilot_queries=args.pilot_queries, + probe_only=args.probe_only, scores={}, orders={}) + exempt = set(exempt_types(entry)) + if exempt: + oracle['exempt_types'] = dict(types=sorted(exempt), reason=EXEMPTION_REASON) + oracle['graphs'] = graphs + oracle['runtime_adjustments'] = (['Set the default tensor device during CQD forward; upstream hybrid padding omits device.'] + if entry['method'] in ('cqd', 'cqd-hybrid') else []) + if entry['method'] == 'qto': + oracle['runtime_adjustments'] = ['Lazy canonical 100-head calibration; up to 100 max-reduction partitions instead of 10.'] + oracle['helpers_sha256'] = {'reference_qto.py': sha(Path(__file__).with_name('reference_qto.py'))} + if entry['method'] == 'clmpt': + oracle['runtime_adjustments'] = ['Partition independent attention heads; preserve complete query sequences and batches.'] + if entry['method'] == 'incoming-relation': + oracle['runtime_adjustments'] = ['Seed each query independently before upstream random tie shuffling; preserve Boolean classes.'] + if entry['method'] in ('ultraquery', 'ultraquery-lp') and entry['options'].get('observed_traversal'): + oracle['runtime_adjustments'] = ['Opt-in observed traversal (not an upstream option): every projection takes the maximum ' + "with upstream's SymbolicTraversal of the same fuzzy set."] + if entry['method'] in ('ultraquery', 'ultraquery-lp') and entry['options'].get('calibration'): + oracle.setdefault('runtime_adjustments', []).append( + 'Opt-in softmax-times-degree calibration (not an upstream option): every projection applies the softmax of ' + "upstream's projection logits times the observed-tail mass of upstream's SymbolicTraversal, capped at 0.9999.") + if entry['method'] in ('gnnqe', 'inductive-gnnqe', 'incoming-relation'): + oracle['runtime_adjustments'].append('Configure TorchDrug size-to-index helper and Python/RDKit imports.') + if entry['method'] in ('cqd', 'cqd-hybrid') and entry['options'].get('reference_batching') is False: + oracle['runtime_adjustments'].append( + 'Bound every CQD atom stage with declared row batches and stage-global normalization before observed overrides; ' + 'use upstream row-wise candidate selection and global padding, streaming final max, and released 3p/4p quirks.') + oracle['arithmetic_profile'] = dict(name='cqd-bounded-rows-v1', row_batch_size=entry['options']['row_batch_size'], + topk='row-wise', normalization='stage-global-before-observed-facts') + oracle['helpers_sha256'] = {'reference_cqd.py': sha(Path(__file__).with_name('reference_cqd.py'))} + elif (entry['method'] in ('cqd', 'cqd-hybrid') and entry['options'].get('final_batch_size') + and args.device.startswith('cuda')): + oracle['runtime_adjustments'].append('Stream final 4p projection with stage-global normalization; preserve released 4p quirks.') + oracle['helpers_sha256'] = {'reference_cqd.py': sha(Path(__file__).with_name('reference_cqd.py'))} + try: + driver = subprocess.check_output(['nvidia-smi', '--query-gpu=uuid,name,driver_version', '--format=csv,noheader'], text=True).strip() + except (OSError, subprocess.CalledProcessError): + driver = None + oracle['environment'] = dict(python=platform.python_version(), torch=str(torch.__version__), cuda=torch.version.cuda, + gpu=torch.cuda.get_device_name(args.device) if args.device.startswith('cuda') else None, + driver=driver, precision='float32 IEEE; no autocast', threads=torch.get_num_threads()) + with torch.no_grad(): + predict = predictor(entry, args.upstream, args.input_root / entry['checkpoint'], n, nr, triples, queries, + args.device, args.pilot_queries if args.probe_only else None) + counts, start = collections.Counter(), 0 + for end in plan['batch_ends']: + keys = plan['queries'][start:end] + shape = queries[keys[0]][0] + if counts[shape] < args.pilot_queries and shape not in exempt: + values = predict(shape, [queries[key][1] for key in keys]) + if args.probe_only: + keys = keys[:args.pilot_queries - counts[shape]] + values = values[:len(keys)] + orders = values.argsort(dim=-1, descending=True) + for key, value, order in zip(keys, values, orders): + oracle['scores'][key], oracle['orders'][key] = value.cpu(), order.cpu() + counts[shape] += len(keys) + print(json.dumps(dict(shape=shape, queries=counts[shape])), flush=True) + start = end + args.output.parent.mkdir(parents=True, exist_ok=True) + torch.save(oracle, args.output) + + +if __name__ == '__main__': + main() diff --git a/benchmarks/cqa/verification/reference_cqd.py b/benchmarks/cqa/verification/reference_cqd.py new file mode 100644 index 000000000..29b226604 --- /dev/null +++ b/benchmarks/cqa/verification/reference_cqd.py @@ -0,0 +1,183 @@ +"""Bound the released 4p oracle's final projection; no DICE imports.""" + +import torch + + +def query_4p(model, source, queries, batch_size): + if len(queries) != 1: + raise ValueError('The released hybrid executor requires single-query batches') + entity, predicate = model.embeddings + + def raw(lhs, rel, rhs): + result = model.score_o(lhs, rel, rhs)[0] + return result.sigmoid() * model.max_norm if model.do_sigmoid else result + + def scoring(lhs, rel, rhs): + result = raw(lhs, rel, rhs) + if model.do_normalize: + result = model.max_norm * (result - result.min()) / (result.max() - result.min()) + return result + + ids = queries[:, 0] + # Each stage gathers by ID, so selected embedding tensors are unnecessary. + for position in (1, 2, 3): + relation = queries[:, position] + atoms = torch.stack((ids, relation.expand_as(ids)), -1) + prefix, _, ids = source.score_candidates( + queries=atoms, filters=model.filters, s_emb=entity(ids), p_emb=predicate(relation), + candidates_emb=entity.weight, k=model.k, max_norm=model.max_norm, max_k=model.max_k, + entity_embeddings=lambda indices: None, scoring_function=scoring) + ids = ids.flatten().long() + + # Preserve both release quirks: r3 embeddings for atom 4, and only atom 3's prefix. + relation = predicate(queries[:, 3]) + prefix = prefix.flatten() + low, high = None, None + if model.do_normalize: + for heads in ids.split(batch_size): + values = raw(entity(heads), relation.expand(len(heads), -1), entity.weight) + lo, hi = values.min(), values.max() + low = lo if low is None else torch.minimum(low, lo) + high = hi if high is None else torch.maximum(high, hi) + result = None + for offset in range(0, len(ids), batch_size): + heads = ids[offset:offset + batch_size] + values = raw(entity(heads), relation.expand(len(heads), -1), entity.weight) + if model.do_normalize: + values = model.max_norm * (values - low) / (high - low) + if model.max_norm != 1: + for row, head in enumerate(heads.tolist()): + values[row, model.filters.get((head, int(queries[0, 4])), [])] = 1 + before = prefix[offset:offset + len(heads), None] + combined = before * values if model.t_norm_name == 'prod' else torch.minimum(before, values) + best = combined.max(0).values + result = best if result is None else torch.maximum(result, best) + return result[None] + + +class _BoundedStages: + """Partition score_o calls while delegating fact overrides/top-k to upstream. + + The profile fixes both GEMM row groups and row-wise candidate selection. + Extrema cover an entire logical atom stage, before any observed-fact override. + Only selected IDs/scores survive between stages, never selected embeddings. + """ + + def __init__(self, model, source, batch_size): + self.model, self.source, self.batch_size = model, source, batch_size + self.entity, self.predicate = model.embeddings + self.combine = torch.mul if model.t_norm_name == 'prod' else torch.minimum + + def raw(self, heads, relation): + values = self.model.score_o(self.entity(heads), self.predicate(relation).expand(len(heads), -1), self.entity.weight)[0] + return values.sigmoid() * self.model.max_norm if self.model.do_sigmoid else values + + def rows(self, heads, relation): + low, high, retained = None, None, None + if self.model.do_normalize: + for group in heads.split(self.batch_size): + values = self.raw(group, relation) + low = values.min() if low is None else torch.minimum(low, values.min()) + high = values.max() if high is None else torch.maximum(high, values.max()) + if len(heads) <= self.batch_size: + retained = values + del values + if high == low: + raise ValueError('Upstream min-max normalization is undefined for constant scores') + for offset in range(0, len(heads), self.batch_size): + group = heads[offset:offset + self.batch_size] + values = retained if retained is not None else self.raw(group, relation) + retained = None + if self.model.do_normalize: + values = self.model.max_norm * (values - low) / (high - low) + yield offset, group, values + + def candidates(self, heads, relation, values, k): + # score_candidates remains responsible for observed-fact replacement, + # per-row beam width, and the upstream topk implementation. + return self.source.score_candidates( + queries=torch.stack((heads, relation.expand_as(heads)), -1), + filters=self.model.filters, s_emb=self.entity(heads), p_emb=self.predicate(relation), + candidates_emb=self.entity.weight, k=k, max_norm=self.model.max_norm, max_k=self.model.max_k, + entity_embeddings=lambda indices: None, scoring_function=lambda *_: values) + + def select(self, heads, relation): + selected = [] + for _, group, values in self.rows(heads, relation): + for index in range(len(group)): + scores, _, ids = self.candidates(group[index:index + 1], relation, values[index:index + 1], self.model.k) + selected.append((scores, ids)) + # Upstream hybrid pads each parent's selections to the largest width + # across the full stage, not to a per-partition width. + width = max(scores.shape[1] for scores, _ in selected) + scores = torch.cat([torch.nn.functional.pad(s, (0, width - s.shape[1])) for s, _ in selected]) + ids = torch.cat([torch.nn.functional.pad(i, (0, width - i.shape[1])) for _, i in selected]) + return scores, ids + + def finish(self, heads, relation, prefix, observed_relation=None): + result = None + for offset, group, values in self.rows(heads, relation): + scores, _, _ = self.candidates(group, relation if observed_relation is None else observed_relation, values, None) + best = self.combine(prefix[offset:offset + len(group), None], scores).max(0).values + result = best if result is None else torch.maximum(result, best) + return result[None] + + def path(self, queries): + heads = queries[:, 0] + if queries.shape[1] == 2: + _, group, values = next(self.rows(heads, queries[:, 1])) + return self.candidates(group, queries[:, 1], values, None)[0] + stages = [] + for position in range(1, queries.shape[1] - 1): + scores, heads = self.select(heads.flatten(), queries[:, position]) + stages.append(scores) + if queries.shape[1] == 5: + # Released 4p discards earlier prefixes and scores r3 again, while + # its observed-fact lookup still uses the requested r4. + return self.finish(heads.flatten(), queries[:, 3], stages[-1].flatten(), queries[:, 4]) + prefix = stages[0].flatten() + if len(stages) == 2: + # Literal upstream 3p repeat (not repeat_interleave). + prefix = self.combine(prefix.repeat(stages[1].shape[1]), stages[1].flatten()) + return self.finish(heads.flatten(), queries[:, -1], prefix) + + +def query_bounded(model, source, shape, queries, batch_size): + """Independent bounded oracle for all eleven released positive shapes. + + This is a declared arithmetic profile, not a claim of bitwise equivalence + with the release's unpartitioned GEMMs. Both oracle and implementation must + use the declared row grouping; numerical/rank/tie checks remain unchanged. + """ + if len(queries) != 1 or type(batch_size) is not int or batch_size < 1: + raise ValueError('Bounded CQD requires one query and a positive integer row batch size') + stages = _BoundedStages(model, source, batch_size) + if shape in ('1p', '2p', '3p', '4p'): + return stages.path(queries) + if shape == 'pi': + return stages.combine(stages.path(queries[:, :3]), stages.path(queries[:, 3:5])) + + # Retain upstream composition for all intersection/union base branches. + def scoring(lhs, rel, rhs): + values = model.score_o(lhs, rel, rhs)[0] + if model.do_sigmoid: + values = values.sigmoid() * model.max_norm + if model.do_normalize: + values = model.max_norm * (values - values.min()) / (values.max() - values.min()) + return values + + union = shape in ('2u', 'up') + function = source.query_2u_dnf if union else getattr(source, 'query_' + ('2i' if shape == 'ip' else shape)) + combine = (lambda a, b: 1 - (1 - a) * (1 - b)) if model.t_norm_name == 'prod' else torch.maximum + base = function(entity_embeddings=model.embeddings[0], predicate_embeddings=model.embeddings[1], + queries=queries[:, :4] if shape in ('ip', 'up') else queries, + filters=model.filters, max_k=model.max_k, max_norm=model.max_norm, + scoring_function=scoring, **{'t_conorm' if union else 't_norm': combine if union else stages.combine}) + if shape not in ('ip', 'up'): + return base + existing = int((base == 1).sum()) if model.max_norm != 1 else 0 + count = min(model.k + existing, model.nentity) + if model.max_k is not None: + count = min(count, model.max_k) + prefix, heads = base.topk(count, dim=1) + return stages.finish(heads.flatten(), queries[:, 5 if union else 4], prefix.flatten()) diff --git a/benchmarks/cqa/verification/reference_kgfm.py b/benchmarks/cqa/verification/reference_kgfm.py new file mode 100644 index 000000000..ff86c4238 --- /dev/null +++ b/benchmarks/cqa/verification/reference_kgfm.py @@ -0,0 +1,159 @@ +"""Independent adapter and fixed-k algebra; atomic backbone logits are shared.""" + +import math +from collections import Counter, defaultdict + +import torch + + +class ReferenceKGFM: + def __init__(self, triples, n, nr, adapter, *, beam_size, device, restore_observed=True, observed_ties='entity', negation='model'): + if (adapter['feature_mode'] not in ('context_scores', 'context_scores_v1', 'global') or adapter['hidden_dim'] + or adapter['normalization'] != 'none' or adapter['observed_mix'] not in (0, 1) + or not 0 <= adapter.get('membership_threshold', 0.) < 1 + or adapter.get('fixed_calibration') not in (None, 'minmax', 'softmax-degree', 'softmax', 'softmax-degree-ties', 'softmax-mass') + or (adapter.get('fixed_calibration') + and (adapter['feature_mode'] != 'global' or any(w for row in adapter['weights'] for w in row)))): + raise ValueError('This reference covers the frozen linear observed-fact recipe') + self.n, self.nr, self.adapter = n, nr, adapter + self.k, self.device = beam_size, device + self.restore_observed = restore_observed + self.observed_ties = observed_ties + self.negation = negation + self.edges = defaultdict(set) + triples = set(map(tuple, triples)) + self.degree = Counter(h for h, _, _ in triples) + self.frequency = Counter(r for _, r, _ in triples) + for h, r, t in triples: + self.edges[h, r].add(t) + self.rows = {} + self.raw_rows = 0 + + def capture(self, conditions, raw): + """Calibrate fresh logits without calling the production feature/adapter code.""" + raw = raw.double() + observed = torch.zeros_like(raw, dtype=torch.bool) + features = [] + for i, (h, r) in enumerate(conditions): + tails = self.edges[h, r] + observed[i, sorted(tails)] = True + features.append([1., math.log1p(len(tails)) / math.log1p(self.n), + math.log1p(self.degree[h]) / math.log1p(self.n * self.nr), + math.log1p(self.frequency[r]) / math.log1p(self.n**2)]) + mean = raw.mean(1) + centered = raw - raw.max(1, keepdim=True).values + mass = centered.exp() + total = mass.sum(1) + entropy = (total.log() - (mass * centered).sum(1) / total) / math.log(self.n) + top = raw.topk(2, dim=1).values + gap = ((top[:, 0] - top[:, 1]) / 4).tanh() + count = observed.sum(1) + contrast = torch.where(count > 0, (((raw * observed).sum(1) / count.clamp_min(1) - mean) / 4).tanh(), 0.) + features = torch.cat((raw.new_tensor(features), + torch.stack(((mean / 4).tanh(), entropy, gap, contrast), dim=1)), 1) + if self.adapter['feature_mode'] == 'global': + features = raw.new_ones((len(raw), 1)) + calibration = self.adapter.get('fixed_calibration') + if calibration == 'minmax': + # CQD-Hybrid: 0.9 (s - min) / (max - min) per row; a constant row gets membership zero everywhere. + low, high = raw.min(1, keepdim=True).values, raw.max(1, keepdim=True).values + spread = high - low + calibrated = torch.where(spread > 0, 0.9 * (raw - low) / torch.where(spread > 0, spread, 1.), 0.).log() + elif calibration in ('softmax-degree', 'softmax', 'softmax-degree-ties'): + # QTO: softmax over tails times the observed tail count (at least one), capped at 0.9999 below one; 'softmax' + # drops the count, '-ties' orders capped tails by the raw score. Masked known tails leave the softmax. + scores = raw + if self.adapter.get('mask_known_logits'): + keep = ~observed | observed.all(1, keepdim=True) + scores = torch.where(keep, raw, raw.new_tensor(-math.inf)) + level = torch.log_softmax(scores, 1) + if calibration != 'softmax': + level = level + count.clamp_min(1).double().log()[:, None] + calibrated = torch.minimum(level, raw.new_tensor(math.log(0.9999))) + if calibration == 'softmax-degree-ties': + calibrated = calibrated - torch.where(level >= math.log(0.9999), 1e-6 * torch.atan2(torch.ones_like(raw), raw) / math.pi, + raw.new_zeros(())) + elif calibration == 'softmax-mass': + # FIT: softmax times the observed tail count over the softmax mass on the observed tails (no factor for a row + # without observed tails), capped at 0.9999 below one. + share = torch.softmax(raw, 1) + mass = (share * observed).sum(1) + factor = torch.where(count > 0, count.double() / mass, raw.new_ones(())) + calibrated = torch.minimum((share * factor[:, None]).log(), raw.new_tensor(math.log(0.9999))) + else: + u, v = (features @ raw.new_tensor(self.adapter['weights']).T).unbind(1) + bound = math.log(self.adapter['scale_bound']) + alpha = (bound * (u * (math.log(2) / bound)).tanh()).exp() + logits = alpha[:, None] * raw + self.adapter['bias_bound'] * v.tanh()[:, None] + calibrated = torch.nn.functional.logsigmoid(logits) + if self.adapter.get('membership_threshold'): + # A fixed threshold makes memberships at or below it exactly zero; observed facts still override. + calibrated = calibrated.masked_fill(calibrated <= math.log(self.adapter['membership_threshold']), -math.inf) + if self.adapter['observed_mix'] == 1: + if self.observed_ties == 'model': + # Known facts a billionth below one, in the order of their calibrated membership. + calibrated = torch.where(observed, -1e-9 * (1 - calibrated.exp()), calibrated) + else: + calibrated = calibrated.masked_fill(observed, 0.) + self.rows.update((tuple(condition), row.cpu()) for condition, row in zip(conditions, calibrated)) + self.raw_rows += len(conditions) + + @staticmethod + def complement(value): + result = torch.empty_like(value) + small = value < -math.log(2) + result[small] = torch.log1p(-value[small].exp()) + result[~small] = torch.log(-torch.expm1(value[~small])) + return result + + def predict(self, query, operator='product'): + """Interpret nested tuples directly, with no production parser or executor.""" + def visit(value): + if isinstance(value, int): + scores = torch.full((self.n,), -torch.inf, dtype=torch.float64, device=self.device) + scores[value] = 0. + return scores, {value}, True + if (len(value) == 2 and value[1] and value[1] != (-1,) + and all(isinstance(r, int) for r in value[1])): + scores, answers, positive = visit(value[0]) + for position, relation in enumerate(value[1]): + if relation == -2: + if self.negation == 'observed': + # Known facts alone: exclude the branch's exact answers, keep every other entity at one. + scores = torch.zeros(self.n, dtype=torch.float64, device=self.device) + scores[sorted(answers)] = -torch.inf + else: + scores = self.complement(scores) + positive = False + answers = set(range(self.n)) - answers + continue + if isinstance(value[0], int) and position == 0: + scores = self.rows[value[0], relation].to(self.device) + else: + heads = scores.argsort(descending=True, stable=True)[:self.k].tolist() + reduced = torch.full_like(scores, -torch.inf) + for head in heads: + if torch.isneginf(scores[head]).item(): + continue + row = self.rows[head, relation].to(self.device) + joined = scores[head] + row if operator == 'product' else torch.minimum(scores[head], row) + reduced = torch.maximum(reduced, joined) + scores = reduced + answers = set().union(*(self.edges[h, relation] for h in answers)) + return scores, answers, positive + union = value[-1] == (-1,) + branches = [visit(branch) for branch in (value[:-1] if union else value)] + rows = torch.stack([row for row, _, _ in branches]) + if operator == 'product': + scores = self.complement(self.complement(rows).sum(0)) if union else rows.sum(0) + else: + scores = rows.max(0).values if union else rows.min(0).values + sets = [answers for _, answers, _ in branches] + answers = set.union(*sets) if union else set.intersection(*sets) + return scores, answers, all(positive for _, _, positive in branches) + + scores, answers, positive = visit(query) + if self.restore_observed and self.adapter['observed_mix'] == 1 and positive: + scores = scores.clone() + scores[sorted(answers)] = 0. + return scores diff --git a/benchmarks/cqa/verification/reference_qto.py b/benchmarks/cqa/verification/reference_qto.py new file mode 100644 index 000000000..4aee4da00 --- /dev/null +++ b/benchmarks/cqa/verification/reference_qto.py @@ -0,0 +1,70 @@ +"""Lazy reference adjacency with upstream QTO's canonical 100-head calibration.""" + +from collections import OrderedDict, defaultdict + +import torch + + +class ReferenceRelations: + def __init__(self, source, model, triples, n, device, threshold, fraction=100, cache_bytes=256 * 2**20): + self.source, self.model = source, model + self.n, self.device, self.threshold, self.fraction = n, device, threshold, fraction + self.cache, self.cache_bytes, self.bytes = OrderedDict(), cache_bytes, 0 + self.observed = defaultdict(lambda: defaultdict(list)) + for head, relation, tail in triples: + self.observed[relation][head].append(tail) + + def block(self, relation, start): + key = relation, start + if key in self.cache: + self.cache.move_to_end(key) + return self.cache[key] + end = min(start + 100, self.n) + heads = torch.arange(start, end, device=self.device) + rel = torch.tensor([relation], device=self.device) + scores = self.source.kge_forward(self.model, heads, rel, self.device, self.n) + degree = scores.new_tensor([max(1, len(self.observed[relation][head])) for head in range(start, end)]) + value = torch.softmax(scores, dim=1) * degree[:, None] + value = ((value >= self.threshold).float() * value).cpu() + value = (value >= 1).float() * .9999 + (value < 1).float() * value + for head in range(start, end): + value[head - start, self.observed[relation][head]] = 1 + size = value.numel() * value.element_size() + if size <= self.cache_bytes: + while self.bytes + size > self.cache_bytes: + _, previous = self.cache.popitem(last=False) + self.bytes -= previous.numel() * previous.element_size() + self.cache[key] = value + self.bytes += size + return value + + def __getitem__(self, relation): + return _Relation(self, int(relation)) + + +class _Relation: + def __init__(self, owner, relation): + self.owner, self.relation = owner, relation + + def __getitem__(self, part): + return _Rows(self.owner, self.relation, part * (self.owner.n // self.owner.fraction)) + + +class _Rows: + def __init__(self, owner, relation, offset): + self.owner, self.relation, self.offset = owner, relation, offset + + def to_dense(self): + # Upstream immediately selects active rows; materialize only that selection. + return self + + def __getitem__(self, key): + indices, columns = key + if columns != slice(None): + raise ValueError('Expected the complete candidate domain') + heads = indices.cpu().tolist() + rows = [] + for index in heads: + head = index + self.offset + rows.append(self.owner.block(self.relation, head // 100 * 100)[head % 100]) + return torch.stack(rows).to(self.owner.device) diff --git a/benchmarks/cqa/verification/verify_kgfm.py b/benchmarks/cqa/verification/verify_kgfm.py new file mode 100644 index 000000000..d6665109e --- /dev/null +++ b/benchmarks/cqa/verification/verify_kgfm.py @@ -0,0 +1,101 @@ +"""Validate the frozen KGFM pipeline against independent calibration/composition.""" + +import json +from pathlib import Path +from unittest.mock import patch + +import torch + +from dicee.query_answering.context import fingerprint +from dicee.query_answering.engine import AtomicScorer, QueryAnswerer + +from ..study import bundle_entry, checksum, entry_inputs_identity, read, run_job, verify_bundle +from .reference_kgfm import ReferenceKGFM + + +def verify(bundle_dir, entry_id, root, output, *, device='cuda', probes=2): + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + if (output / 'pilot').exists(): + raise FileExistsError('Use a fresh directory so every prediction is checked') + bundle = verify_bundle(bundle_dir, root) + entry = bundle_entry(bundle, entry_id) + if not entry['method'].endswith('-adapter') or len(entry['adapters']) != 1: + raise ValueError('Select a frozen single-operator KGFM entry') + adapter = read(Path(root) / next(iter(entry['adapters'].values()))) + observed_facts = entry['options'].get('observed_facts') + if observed_facts is not None: + adapter['observed_mix'] = float(observed_facts != 'none') + references, scorers, comparisons, scores, orders = {}, {}, {}, {}, {} + original = AtomicScorer.rows + original_init = QueryAnswerer.__init__ + + def initialize(engine, *args, **kwargs): + original_init(engine, *args, **kwargs) + scorer = engine.scorer + variant = 'without-adapter' if engine.adapter.metadata.get('calibration') == 'identity' else 'learned' + expected_adapter = adapter if variant == 'learned' else dict(adapter, feature_mode='global', weights=[[0.], [0.]], + membership_threshold=0., fixed_calibration=None) + reference = ReferenceKGFM(scorer.context.triples, scorer.n, scorer.nr, expected_adapter, + beam_size=entry['options']['beam_size'], device=scorer.device, + restore_observed=observed_facts in (None, 'both'), + observed_ties=entry['options'].get('observed_ties', 'entity'), + negation=entry['options'].get('negation', 'model')) + scorers[id(scorer)] = reference + references[variant] = reference + + def capture(scorer, conditions): + raw = original(scorer, conditions) + scorers[id(scorer)].capture(conditions, raw) + return raw + + def compare(query, actual, variant='learned'): + key = query.identity + expected = references[variant].predict(query.query, entry['operators'][query.shape]) + order = expected.argsort(descending=True) + finite = actual.isfinite() & expected.isfinite() + error = (actual[finite] - expected[finite]).abs().max().item() if finite.any() else 0. + result = dict(shape=query.shape, variant=variant, max_error=error, + scores_match=torch.allclose(actual, expected, atol=1e-12, rtol=1e-12), + ranks_match=torch.equal(actual.argsort(descending=True), order), + ties_match=torch.equal(actual[order][1:] == actual[order][:-1], + expected[order][1:] == expected[order][:-1])) + result['passed'] = all(result[name] for name in ('scores_match', 'ranks_match', 'ties_match')) + comparisons[f'{variant}:{key}'] = result + scores.setdefault(variant, {})[key], orders.setdefault(variant, {})[key] = expected.cpu(), order.cpu() + print(json.dumps(dict(query=len(comparisons), **result)), flush=True) + + with patch.object(AtomicScorer, 'rows', capture), patch.object(QueryAnswerer, '__init__', initialize): + report = run_job(bundle_dir, entry_id, root, output / 'pilot', device=device, phase='pilot', + pilot_queries=probes, probe_only=True, on_prediction=compare, + on_control_prediction=lambda query, actual: compare(query, actual, 'without-adapter')) + variants = {'learned', 'without-adapter'} if entry.get('adapter_ablation') else {'learned'} + if set(references) != variants or any({row['shape'] for row in comparisons.values() if row['variant'] == variant} + != set(entry['query_types']) for variant in variants): + raise ValueError('Integration check missed a query type or calibration variant') + identity = dict(entry_sha256=fingerprint({k: v for k, v in entry.items() if k != 'verification'}), + inputs_sha256=entry_inputs_identity(bundle, entry)) + scope = ('Independent context features, linear adapter, tuple interpretation, stable fixed-k pruning, ' + 'fuzzy composition and positive observed-answer restoration. Shared native backbone logits; ' + 'this is integration evidence, not an independent backbone reproduction. ' + 'Paired entries also verify the identity-calibration control on every query type.') + oracle = dict(version=1, **identity, validation_plan_sha256=bundle['plans'][f'{entry_id}/valid']['sha256'], + pilot_queries=probes, probe_only=True, scores=scores['learned'], orders=orders['learned'], + comparison_scores={key: value for key, value in scores.items() if key != 'learned'}, + comparison_orders={key: value for key, value in orders.items() if key != 'learned'}, + environment=report['benchmark_run']['environment'], runtime_adjustments=[scope]) + torch.save(oracle, output / 'reference.pt') + evidence = dict(**identity, source_sha256=bundle['source_sha256'], environment=report['benchmark_run']['environment'], + reference_sha256=checksum(output / 'reference.pt'), reference_commit=None, + reference_implementation={p.name: checksum(p) for p in (Path(__file__), Path(__file__).with_name('reference_kgfm.py'))}, + queries=comparisons, scope=scope, probe_only=True, + reference_adjustments=[scope], atol=1e-12, rtol=1e-12, + passed=all(row['passed'] for row in comparisons.values()), + raw_rows=sum(reference.raw_rows for reference in references.values()), + unique_rows=sum(len(reference.rows) for reference in references.values()), + peak_cuda_allocated_bytes=report['peak_cuda_allocated_bytes'], + total_wall_seconds=report['total_wall_seconds']) + (output / 'integration.json').write_text(json.dumps(evidence, indent=2) + '\n') + if not evidence['passed']: + raise RuntimeError('Integration parity failed; inspect integration.json') + return evidence diff --git a/benchmarks/kgfm_inference.py b/benchmarks/kgfm_inference.py index 342e26026..b4911392a 100644 --- a/benchmarks/kgfm_inference.py +++ b/benchmarks/kgfm_inference.py @@ -15,12 +15,17 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -CHECKPOINTS = {'ULTRA': 'ultra_3g.pth', 'TRIX': 'trix/entity_prediction.pth', 'Flock': 'flock/flock_entity.pth'} +CHECKPOINTS = {'ULTRA': 'ultra_3g.pth', 'TRIX': 'trix/entity_prediction.pth', 'Flock': 'flock/flock_entity.pth', + 'KGICL': 'kgicl/KG-ICL-6L/model_best.tar'} def digest(path): - with path.open('rb') as stream: - return hashlib.file_digest(stream, 'sha256').hexdigest() + # Chunked hashing also runs on the Python 3.9 reference environment of official KG-ICL. + result = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(1 << 20), b''): + result.update(block) + return result.hexdigest() def inference_arguments(args): @@ -41,7 +46,7 @@ def worker(args): import torch from dicee.evaluation.link_prediction import evaluate_lp - from dicee.models import TRIX, ULTRA, Flock + from dicee.models import KGICL, TRIX, ULTRA, Flock torch.set_num_threads(args.threads) torch.manual_seed(42) @@ -71,8 +76,8 @@ def worker(args): graph_inference_backend=args.inference_backend, graph_relation_cache_mb=args.relation_cache_mb, graph_projection_cache_mb=args.projection_cache_mb, graph_inference_compile=args.compile_inference, flock_compact_state=args.compact_state, flock_compile_sampler=args.compile_sampler, - flock_pack_walks=args.pack_walks) - cls = {'ULTRA': ULTRA, 'TRIX': TRIX, 'Flock': Flock}[args.model] + flock_pack_walks=args.pack_walks, kgicl_masked_distances=metadata.get('kgicl_masked_distances')) + cls = {'ULTRA': ULTRA, 'TRIX': TRIX, 'Flock': Flock, 'KGICL': KGICL}[args.model] begin = time.perf_counter() model = cls(settings).load_pretrained(checkpoint).set_graph(facts).eval().requires_grad_(False).to(device=args.device, dtype=getattr(torch, args.dtype)) setup_seconds = time.perf_counter() - begin diff --git a/benchmarks/kgfm_upstream.py b/benchmarks/kgfm_upstream.py index c0b8b6cf8..46f4cf35a 100644 --- a/benchmarks/kgfm_upstream.py +++ b/benchmarks/kgfm_upstream.py @@ -4,6 +4,13 @@ environment. Dataset loading, graph construction, filtering, and fixed-walk transfers are outside warm inference timings. Flock additionally replays the official sampler's records through both neural models for a score parity check. + +KG-ICL's official worker runs in its own environment (``--upstream-python``) +on the pinned checkout plus tests/fixtures/kgicl/upstream-fixes.patch, unless +``--upstream-variant unpatched`` is given. Both models read the same graph, +the same DICE-sampled prompt graphs, written in the official case format, and +the same answer-distance mask. Prompt graph extraction is preprocessing and +is not timed for either implementation. """ import argparse import importlib.metadata @@ -24,8 +31,10 @@ 'ULTRA': '427966ad8ed60420eef034063d44f3153addff90', 'TRIX': '7596e14eefefe89e61396205a0550172cadeddb0', 'Flock': 'f35103d25a78bdf4075de5c673a51de4979aa4d7', + 'KGICL': '6a3166e347ae468acdfb30a70a2cf3608b66b8f1', } -SOURCE_DIRS = {'ULTRA': 'ultra', 'TRIX': 'src', 'Flock': 'src_entity'} +SOURCE_DIRS = {'ULTRA': 'ultra', 'TRIX': 'src', 'Flock': 'src_entity', 'KGICL': 'src'} +KGICL_PATCH = ROOT / 'tests' / 'fixtures' / 'kgicl' / 'upstream-fixes.patch' def git(root, *args): @@ -63,11 +72,14 @@ def filtered_metrics(scores, triples, directory): **{f'H@{k}': (ranks <= k).double().mean().item() for k in (1, 3, 10)}} -def gpu_processes(): +def gpu_processes(require=True): if not shutil.which('nvidia-smi'): raise RuntimeError('nvidia-smi is required to detect competing CUDA jobs') run = subprocess.run(['nvidia-smi', '--query-compute-apps=pid,process_name,used_memory', - '--format=csv,noheader,nounits'], check=True, text=True, capture_output=True) + '--format=csv,noheader,nounits'], check=require, text=True, capture_output=True) + if run.returncode: + # Explicitly allowed (--allow-unchecked-gpu), e.g. after an NVML driver/library mismatch. + return ['unavailable: ' + (run.stdout + run.stderr).strip()] lines = [line for line in run.stdout.splitlines() if line.strip()] desktop = {'/usr/bin/kwin_wayland', '/usr/bin/kwin_x11', '/usr/lib/Xorg', '/usr/bin/Xorg'} other = [line for line in lines if int(line.split(',')[0]) != os.getpid() @@ -77,6 +89,91 @@ def gpu_processes(): return lines +def kgicl_data(args, facts, valid, triples, metadata): + """Write the splits and DICE's prompt graphs in the official processed layout. + + The test background is the training graph DICE attaches. Examples are those + DICE samples, so both models read identical prompts. Returns the distance + mask both models apply: hop distances holding under 2% of validation facts. + """ + sys.path.insert(0, str(ROOT)) + from dicee.models import KGICL + from dicee.models.kgicl_prompts import answer_distance_rates + + nr = metadata['num_relations'] + model = KGICL(dict(num_entities=metadata['num_entities'], num_relations=nr, kgicl_prompt_seed=args.prompt_seed)) + model.set_graph(facts) + data = args.output / 'kgicl-data' + queried = set(triples[:, 1].tolist()) + + def write(path, rows): + path.write_text(''.join('\t'.join(map(str, row)) + '\n' for row in rows)) + + for split, queries in (('valid', valid), ('test', triples)): + directory = data / split + directory.mkdir(parents=True, exist_ok=True) + write(directory / 'background.txt', facts.tolist()) + write(directory / 'facts.txt', queries.tolist()) + write(directory / 'filter.txt', torch_cat(facts, valid, triples).tolist()) + write(directory / 'entity2id.txt', [(f'e{i}', i) for i in range(metadata['num_entities'])]) + write(directory / 'relation2id.txt', [(f'r{i}', i) for i in range(nr)]) + for relation in range(nr): + cases = directory / 'cases' / f'r{relation}' + cases.mkdir(parents=True, exist_ok=True) + if relation not in queried: + continue + for index, example in enumerate(model.sampler.examples(relation)): + local = example.triples.tolist() + first = [0, relation, 0 if example.self_loop else 1] + local.remove(first) + write(cases / f'{index}.triples', [first, *local]) + write(cases / f'{index}.labels', [(node, *label) for node, label in enumerate(example.labels.tolist())]) + write(cases / f'{index}.node2id', [(entity, node) for node, entity in enumerate(example.entities.tolist())]) + rates = answer_distance_rates(valid, facts, metadata['num_entities']) + masked = [d for d in range(7) if rates[d] < 0.02] + (data / 'distance_mask.json').write_text(json.dumps(dict(rates=rates, masked_distances=masked)) + '\n') + return masked + + +def torch_cat(*parts): + import torch + return torch.cat(parts) + + +def official_kgicl(args, metadata): + """Official EntityEncoder and DataLoader, pinned and optionally patched, on the written data.""" + import tempfile + + import torch + work = Path(tempfile.mkdtemp(prefix='kgicl-')) + # The official loader derives split paths by replacing these words in the path. + assert not any(word in str(work) for word in ('train', 'valid', 'test')) + for name in ('src', 'run_your_own_dataset'): + shutil.copytree(args.upstream_root / name, work / name) + if args.upstream_variant == 'patched': + subprocess.run(['patch', '-p1', '-s', '-d', str(work), '-i', str(KGICL_PATCH)], check=True) + shutil.copytree(args.output / 'kgicl-data', work / 'data') + sys.path.insert(0, str(work / 'src')) + from types import SimpleNamespace + + from data_loader import DataLoader + from encoder.EntityEncoder import EntityEncoder + + options = SimpleNamespace( + finetune=False, train_batch_size=8, test_batch_size=args.query_batch_size, shot=5, device=torch.device(args.device), + use_attn=True, attn_type='Sigmoid', AGG='max', AGG_rel='max', MSG='concat', use_augment=False, + use_token_set=True, use_prompt_graph=True, prompt_graph_type='all', path_hop=3, hidden_dim=32, attn_dim=5, + n_relation_encoder_layer=3, n_layer=6, act='idd', dropout=0.0, relation_mask_rate=0.0, use_rspmm=False) + loader = DataLoader(options, str(work / 'data' / 'test') + '/', metadata['dataset']) + expected = json.loads((work / 'data' / 'distance_mask.json').read_text())['rates'] + assert all(abs(loader.kg.answer_distance[d] - expected[d]) < 1e-12 for d in range(7)), 'distance rates differ' + model = EntityEncoder(options) + state = torch.load(ROOT / 'checkpoints' / CHECKPOINTS['KGICL'], map_location='cpu', weights_only=True)['state_dict'] + # Official evaluation drops the checkpoint's unused NBFNet keys the same way (strict=False). + model.load_state_dict({key: value for key, value in state.items() if key in model.state_dict()}, strict=True) + return model, loader + + def official_model(args, facts, metadata): import torch from torch_geometric.data import Data @@ -139,16 +236,23 @@ def worker(args): triples = triples_cpu.to(args.device) checkpoint = ROOT / 'checkpoints' / CHECKPOINTS[args.model] started = time.perf_counter() - if args.variant == 'official': + loader = None + if args.variant == 'official' and args.model == 'KGICL': + model, loader = official_kgicl(args, metadata) + elif args.variant == 'official': model, graph = official_model(args, facts, metadata) else: sys.path.insert(0, str(ROOT)) - from dicee.models import TRIX, ULTRA, Flock + from dicee.models import KGICL, TRIX, ULTRA, Flock - cls = {'ULTRA': ULTRA, 'TRIX': TRIX, 'Flock': Flock}[args.model] + cls = {'ULTRA': ULTRA, 'TRIX': TRIX, 'Flock': Flock, 'KGICL': KGICL}[args.model] + masked = None + if args.model == 'KGICL': + masked = json.loads((args.output / 'kgicl-data' / 'distance_mask.json').read_text())['masked_distances'] model = cls(dict(num_entities=metadata['num_entities'], num_relations=metadata['num_relations'], **{args.model.lower() + '_query_batch_size': args.query_batch_size}, - flock_walk_num=args.walk_num, flock_test_samples=1, flock_seed=args.seed)) + flock_walk_num=args.walk_num, flock_test_samples=1, flock_seed=args.seed, + kgicl_prompt_seed=args.prompt_seed, kgicl_masked_distances=masked)) model.load_pretrained(checkpoint).set_graph(facts) model.eval().requires_grad_(False).to(device=args.device, dtype=getattr(torch, args.dtype)) initial_weights = {name: value.detach().cpu().clone() for name, value in model.state_dict().items()} @@ -176,6 +280,9 @@ def double_ones(*a, **kw): kw['dtype'] = torch.float64 return original_ones(*a, **kw) torch.ones = double_ones + if loader is not None: + # Official KG-ICL's incidence matrices only select neighbors; they multiply default-dtype one-hots. + loader.M_sub, loader.M_obj = loader.M_sub.double(), loader.M_obj.double() records = {} if args.mode == 'replay': for i in range(len(triples)): @@ -193,6 +300,15 @@ def predict(count=None, capture=False): if args.variant == 'dice' and args.mode == 'native': return torch.stack((model.forward_k_vs_all(query[:, :2]), model.forward_k_vs_all_heads(query[:, 1:])), 1) + if loader is not None: + # Official KG-ICL: tail queries (h, r) and head queries (t, r + R) as inverse tail queries. + sides = [] + for heads, relations in ((query[:, 0], query[:, 1]), (query[:, 2], query[:, 1] + metadata['num_relations'])): + sides.append(torch.cat([model(heads[s:s + args.query_batch_size].cpu().numpy(), + relations[s:s + args.query_batch_size].cpu().numpy(), + loader=loader, training=False)[0] + for s in range(0, len(query), args.query_batch_size)])) + return torch.stack(sides, 1) outputs = [] for side in ('tail', 'head'): chunks = [] @@ -232,7 +348,7 @@ def timed(): torch.cuda.synchronize(args.device) return time.perf_counter() - begin, scores - snapshots = [gpu_processes()] + snapshots = [gpu_processes(not args.allow_unchecked_gpu)] if args.scores_only: with torch.no_grad(): scores = predict().cpu() @@ -253,11 +369,11 @@ def timed(): baseline_memory = torch.cuda.memory_allocated(args.device) / 2**20 timings = [] for _ in range(args.repeats): - snapshots.append(gpu_processes()) + snapshots.append(gpu_processes(not args.allow_unchecked_gpu)) elapsed, scores = timed() timings.append(elapsed) del scores - snapshots.append(gpu_processes()) + snapshots.append(gpu_processes(not args.allow_unchecked_gpu)) peak = torch.cuda.max_memory_allocated(args.device) / 2**20 scores = predict().cpu() if args.variant == 'official' and args.model == 'Flock' and args.mode == 'native' and args.replay_queries: @@ -277,6 +393,9 @@ def timed(): if not hasattr(extension, 'rspmm_add_mul_forward_cuda'): raise RuntimeError('The official fused CUDA extension is missing') dependencies['rspmm_binary_sha256'] = digest(Path(extension.__file__)) + if args.model == 'KGICL' and args.variant == 'official': + source['patch'] = None if args.upstream_variant == 'unpatched' else dict(path=str(KGICL_PATCH.relative_to(ROOT)), + sha256=digest(KGICL_PATCH)) report = dict(model=args.model, variant=args.variant, mode=args.mode, source=source, dataset=metadata['dataset'], test_indices=indices.tolist(), test_triples=len(triples), ranked_queries=2 * len(triples), query_batch_size=args.query_batch_size, @@ -300,13 +419,14 @@ def timed(): print(json.dumps({key: report[key] for key in ['model', 'variant', 'mode', 'median_forward_seconds', 'metrics']}), flush=True) -def compare(directory, stochastic=False, fp64_path=None, raise_on_failure=True): +def compare(directory, stochastic=False, fp64_path=None, raise_on_failure=True, same_environment=True): import torch official, dice = [json.loads((directory / f'{name}.json').read_text()) for name in ('official', 'dice')] for key in ('test_indices', 'query_batch_size', 'walk_num', 'test_samples', 'seed', 'torch', 'cuda', 'dtype', 'tf32', 'hardware', 'device', 'checkpoint_sha256', 'train_sha256', 'test_sha256'): - assert official[key] == dice[key], key + if same_environment or key not in ('torch', 'cuda'): + assert official[key] == dice[key], key left, right = [torch.load(directory / f'{name}.pt', weights_only=True) for name in ('official', 'dice')] assert torch.isfinite(left).all() and torch.isfinite(right).all() differences = {key: dice['metrics'][key] - value for key, value in official['metrics'].items()} @@ -351,6 +471,12 @@ def main(): parser.add_argument('--seed', type=int, default=42) parser.add_argument('--walk-num', type=int, default=128) parser.add_argument('--replay-queries', type=int, default=8) + parser.add_argument('--upstream-python', type=Path, help='Interpreter of the official model (KG-ICL: its own environment)') + parser.add_argument('--upstream-variant', choices=['patched', 'unpatched'], default='patched', + help='KG-ICL: apply tests/fixtures/kgicl/upstream-fixes.patch to the official code (default)') + parser.add_argument('--prompt-seed', type=int, default=0, help='KG-ICL prompt sampling seed') + parser.add_argument('--allow-unchecked-gpu', action='store_true', + help='Record, instead of rejecting, an unavailable nvidia-smi process check') parser.add_argument('--variant', choices=['official', 'dice'], help=argparse.SUPPRESS) parser.add_argument('--mode', choices=['native', 'replay'], default='native', help=argparse.SUPPRESS) parser.add_argument('--walks', type=Path, help=argparse.SUPPRESS) @@ -384,13 +510,26 @@ def main(): for mode, queries in modes: directory = args.output / mode directory.mkdir(parents=True, exist_ok=True) + if args.model == 'KGICL': + import numpy as np + import torch + metadata = json.loads((args.indexed_data / 'result.json').read_text()) + facts = torch.from_numpy(np.load(args.indexed_data / 'train_set.npy').astype(np.int64)) + valid = torch.from_numpy(np.load(args.indexed_data / 'valid_set.npy').astype(np.int64)) + test = torch.from_numpy(np.load(args.indexed_data / 'test_set.npy').astype(np.int64)) + indices = torch.randperm(len(test), generator=torch.Generator().manual_seed(args.seed))[:queries] + kgicl_data(argparse.Namespace(output=directory, prompt_seed=args.prompt_seed), facts, valid, test[indices], metadata) + def run_worker(variant, **overrides): - command = [sys.executable, str(Path(__file__).resolve())] + python = args.upstream_python if variant == 'official' and args.upstream_python else sys.executable + command = [str(python), str(Path(__file__).resolve())] options = vars(args) | dict(mode=mode, queries=queries, output=directory, variant=variant) | overrides for key, value in options.items(): - if key == 'scores_only': + if key in ('scores_only', 'allow_unchecked_gpu'): if value: - command.append('--scores-only') + command.append('--' + key.replace('_', '-')) + continue + if value is None: continue command.extend(['--' + key.replace('_', '-'), str(value)]) label = 'official-fp64' if options['scores_only'] else variant @@ -400,7 +539,7 @@ def run_worker(variant, **overrides): for variant in ('official', 'dice'): run_worker(variant) stochastic = args.model == 'Flock' and mode == 'native' - result = compare(directory, stochastic=stochastic, raise_on_failure=False) + result = compare(directory, stochastic=stochastic, raise_on_failure=False, same_environment=args.upstream_python is None) if not result['scores_close'] and args.model != 'Flock': run_worker('official', dtype='float64', scores_only=True) compare(directory, fp64_path=directory / 'official-fp64.pt') diff --git a/benchmarks/kgfm_zero_shot.py b/benchmarks/kgfm_zero_shot.py index 85c88c4c5..76b0a1c05 100644 --- a/benchmarks/kgfm_zero_shot.py +++ b/benchmarks/kgfm_zero_shot.py @@ -27,13 +27,15 @@ from dicee.evaluation._graph_inference import GraphRankPlan from dicee.evaluation.link_prediction import evaluate_lp from dicee.knowledge_graph import KG -from dicee.models import TRIX, ULTRA, Flock +from dicee.models import KGICL, TRIX, ULTRA, Flock +from dicee.models.kgicl_prompts import answer_distance_rates -MODELS = {"ULTRA": ULTRA, "TRIX": TRIX, "Flock": Flock} +MODELS = {"ULTRA": ULTRA, "TRIX": TRIX, "Flock": Flock, "KGICL": KGICL} CHECKPOINTS = { "ULTRA": "checkpoints/ultra_3g.pth", "TRIX": "checkpoints/trix/entity_prediction.pth", "Flock": "checkpoints/flock/flock_entity.pth", + "KGICL": "checkpoints/kgicl/KG-ICL-6L/model_best.tar", } METRICS = ("MRR", "H@1", "H@3", "H@10") @@ -85,6 +87,12 @@ def check_split_overlap(train, valid, test, *, allow=False): def pretraining_status(model, variant, dataset): + if model == "KGICL": + # KG-ICL pretrains on FB V1, NELL V1 and CoDEx-small. GraIL's FB/NELL splits are + # subgraphs of FB15k-237 and NELL-995, so those graphs share facts with pretraining. + if dataset in ("CoDEx-Small", "FB237_v1", "NELL_v1"): + return "yes" + return "related" if dataset == "FB15k-237" or dataset.startswith("NELL-995") else "no" dataset = {"FB15k-237": "FB15k237", "CoDEx-Medium": "CoDExMedium", "YAGO3-10": "YAGO310", "NELL-995": "NELL995"}.get(dataset, dataset) if model == "ULTRA" and variant in ("4g", "50g") and dataset.startswith("NELL-995-"): @@ -123,6 +131,10 @@ def main(): parser.add_argument('--pack-walks', action=argparse.BooleanOptionalAction, default=True) parser.add_argument('--reuse-queries', action=argparse.BooleanOptionalAction, default=True) parser.add_argument('--prefetch-walks', action=argparse.BooleanOptionalAction, default=True) + parser.add_argument('--prompt-seed', type=int, default=0, help='KG-ICL prompt sampling seed') + parser.add_argument('--distance-mask', action='store_true', + help="KG-ICL: zero entities first reached at distances where under 2%% of validation " + "facts lie (the official evaluation's default heuristic)") parser.add_argument("--seed", type=int, default=42) parser.add_argument("--tie-policy", choices=TIE_POLICIES, default="sort") parser.add_argument("--tie-seed", type=int, default=None, @@ -131,6 +143,8 @@ def main(): args = parser.parse_args() if args.model != "ULTRA" and args.ultra_checkpoint != "3g": parser.error("--ultra-checkpoint applies only to ULTRA") + if args.model != "KGICL" and (args.distance_mask or args.prompt_seed): + parser.error("--distance-mask and --prompt-seed apply only to KGICL") if args.tie_seed is None: args.tie_seed = args.seed if min(args.batch_size, args.query_batch_size, args.threads, args.walk_num, args.test_samples) < 1: @@ -202,6 +216,11 @@ def main(): if overlap["train_test"]: logging.warning("Original-split evaluation includes %d distinct test facts in the " "inference graph; these results contain leakage", overlap["train_test"]) + masked = None + if args.distance_mask: + rates = answer_distance_rates(kg.valid_set, kg.train_set, kg.num_entities) + masked = [d for d in range(7) if rates[d] < 0.02] + write_json(output / "distance_mask.json", {"rates": rates, "masked_distances": masked, "threshold": 0.02}) settings = dict(num_entities=kg.num_entities, num_relations=kg.num_relations, **{f"{args.model.lower()}_query_batch_size": args.query_batch_size}, flock_walk_num=args.walk_num, flock_test_samples=args.test_samples, @@ -209,7 +228,8 @@ def main(): graph_inference_backend=args.inference_backend, graph_relation_cache_mb=args.relation_cache_mb, graph_projection_cache_mb=args.projection_cache_mb, graph_inference_compile=args.compile_inference, flock_compact_state=args.compact_state, flock_compile_sampler=args.compile_sampler, - flock_pack_walks=args.pack_walks) + flock_pack_walks=args.pack_walks, kgicl_prompt_seed=args.prompt_seed, + kgicl_masked_distances=masked) model = MODELS[args.model](settings).load_pretrained(checkpoint) model.set_graph(kg.train_set).eval().requires_grad_(False) initial_weights = {key: value.clone() for key, value in model.state_dict().items()} @@ -264,6 +284,7 @@ def main(): "split_overlap": overlap, "test_facts_in_inference_graph": bool(overlap["train_test"]), "model_label": model_label, "pretraining_status": pretraining, "target_graph_in_pretraining": {"yes": True, "no": False}.get(pretraining), + "kgicl_masked_distances": masked, "test_triples": len(kg.test_set), "ranked_queries": 2 * len(kg.test_set), "num_entities": kg.num_entities, "num_relations": kg.num_relations, "inference_edges": model.edge_type.numel(), "metrics": metrics, diff --git a/benchmarks/plus_h/baselines.json b/benchmarks/plus_h/baselines.json new file mode 100644 index 000000000..a6facaac2 --- /dev/null +++ b/benchmarks/plus_h/baselines.json @@ -0,0 +1,359 @@ +{ + "version": 1, + "answer_filter": "corrected", + "data_root": "KGs/query-benchmarks-plus-h", + "seed": 0, + "threads": 4, + "archives": { + "models": {"sha256": "d30b77a9c84d28de766bb5722c600bdc3c1e46d16cdaa3066ed4faca2b9b236f", "bytes": 4507216854, "source": "https://github.com/april-tools/is-cqa-complex#pre-trained-models"}, + "datasets": { + "source": "https://github.com/april-tools/is-cqa-complex/releases/tag/benchs-1.0", + "checksum_status": "Official release API digest; extracted runtime files are independently pinned by freeze.", + "sha256": "bf60135d7b1bc109ab9d90892e318fa96ace905fdc8204afd270b1fd98002a80", + "bytes": 1675873193 + } + }, + "recipes": [ + { + "defaults": { + "id": "cone-{dataset}", + "method": "cone", + "options": {"center_reg": 0.02}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni", "4p", "4i"], + "query_batch_size": 4, + "query_order": "upstream-pickle", + "reference": { + "status": "native reference port; full reproduction pending", + "settings": "https://arxiv.org/html/2410.12537v3#A6", + "notes": ["Query batch size 4 follows the released checkpoint configuration."], + "graph_independent": true + }, + "blockers": [], + "inference_graph": "train" + }, + "entries": [ + {"dataset": "FB15k237+H", "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/ConE/FB15k237/checkpoint"}, + {"dataset": "NELL995+H", "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/ConE/NELL995/checkpoint"}, + {"dataset": "ICEWS18+H", "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/ConE/ICEWS18/checkpoint"} + ] + }, + { + "defaults": { + "id": "gnnqe-{dataset}", + "method": "gnnqe", + "options": {"logic": "product", "edge_batch_size": 65536}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni", "4p", "4i"], + "query_order": "upstream-pickle", + "reference": { + "status": "native reference port; full reproduction pending", + "settings": "https://arxiv.org/html/2410.12537v3#A6", + "notes": ["Inference preserves the released query batch sizes, sparse reduction order and normalization arithmetic.", "Validation GPU scores, orders and ties match the pinned upstream batches."], + "inference_graph": "train" + }, + "blockers": [], + "inference_graph": "train+valid" + }, + "entries": [ + {"dataset": "FB15k237+H", "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/GNN-QE/model/model_FB15k237.pth", "query_batch_size": 48}, + {"dataset": "NELL995+H", "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/GNN-QE/model/model_NELL995.pth", "query_batch_size": 8}, + {"dataset": "ICEWS18+H", "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/GNN-QE/model/model_ICEWS18.pth", "query_batch_size": 48} + ] + }, + { + "defaults": { + "id": "ultraquery-{dataset}", + "method": "ultraquery", + "checkpoint": "Experiments/query-baselines/upstream/ultra/ckpts/ultraquery.pth", + "options": {"logic": "product", "threshold": 0.0, "cache_bytes": 67108864}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni", "4p", "4i"], + "query_batch_size": 32, + "query_order": "upstream-pickle", + "reference": { + "status": "native reference port; full reproduction pending", + "settings": "https://arxiv.org/html/2410.12537v3#A6", + "notes": [ + "The +H model archive README explicitly references this public PyG checkpoint.", + "Inference preserves the released query batch sizes, sparse reduction order and normalization arithmetic.", + "Validation GPU scores, orders and ties match the pinned upstream batches." + ], + "inference_graph": "train" + }, + "blockers": [], + "inference_graph": "train+valid" + }, + "entries": [{"dataset": "FB15k237+H"}, {"dataset": "NELL995+H"}, {"dataset": "ICEWS18+H"}] + }, + { + "defaults": { + "id": "clmpt-{dataset}", + "method": "clmpt", + "options": {"aggregation": "mean", "candidate_batch_size": 1024}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni", "4p", "4i"], + "query_order": "upstream-pickle", + "reference": { + "status": "native reference port; full reproduction pending", + "settings": "https://arxiv.org/html/2410.12537v3#A6", + "notes": [ + "Batch sizes follow the original CLMPT README/defaults; query grouping is reconstructed using the released converter and +H pickle iteration order.", + "Post-norm predictions depend on batch membership. Both up branches use the original relation groundings.", + "Conversion uses the released query files and pinned batches.", + "Independent attention heads are evaluated separately to bound workspace while preserving complete query batches." + ], + "graph_independent": true + }, + "blockers": [], + "inference_graph": "train" + }, + "entries": [ + {"dataset": "FB15k237+H", "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CLMPT/CLMPT-fb15k237.ckpt", "query_batch_size": 5000}, + {"dataset": "NELL995+H", "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CLMPT/CLMPT-nell995.ckpt", "query_batch_size": 8}, + {"dataset": "ICEWS18+H", "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CLMPT/CLMPT-icews18.ckpt", "query_batch_size": 5000} + ] + }, + { + "defaults": { + "id": "cqd-{dataset}", + "method": "cqd", + "options": { + "beam_size": 2, + "tnorm": "prod", + "max_k": 512, + "normalize": true, + "reference_quirks": true, + "row_batch_size": 32, + "cache_bytes": 536870912, + "per_shape": { + "2p": {"tnorm": "prod"}, + "3p": {"tnorm": "prod"}, + "2i": {"tnorm": "prod"}, + "3i": {"tnorm": "prod"}, + "pi": {"tnorm": "prod"}, + "ip": {"tnorm": "prod"}, + "2u": {}, + "up": {"tnorm": "min"}, + "4p": {"beam_size": 2, "tnorm": "prod"}, + "4i": {"beam_size": 2, "tnorm": "prod"} + }, + "reference_batching": true, + "atomic_negation": true + }, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni", "4p", "4i"], + "query_batch_size": 1, + "query_order": "upstream-pickle", + "reference": {"status": "native reference port; full reproduction pending", "settings": "https://arxiv.org/html/2410.12537v3#A6", "graph_independent": true}, + "blockers": [], + "inference_graph": "train" + }, + "entries": [ + { + "dataset": "FB15k237+H", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/fb15k237checkpoint", + "options": { + "per_shape": { + "2p": {"beam_size": 512}, + "3p": {"beam_size": 8}, + "2i": {"beam_size": 128}, + "3i": {"beam_size": 128}, + "pi": {"beam_size": 256}, + "ip": {"beam_size": 64}, + "2u": {"beam_size": 512, "tnorm": "prod"}, + "up": {"beam_size": 512} + } + }, + "reference": { + "notes": [ + "All 16 types are enabled, including the five negated types using CQD-A signed-atom negation with +H scoring.", + "Released 3p/4p quirks are preserved.", + "FB 2u uses product: full-query CPU diagnostics reproduce published 40.1/42.2, while the appendix states min. The up setting uses the documented min." + ] + } + }, + { + "dataset": "NELL995+H", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/nellcheckpoint", + "options": { + "per_shape": { + "2p": {"beam_size": 512}, + "3p": {"beam_size": 2}, + "2i": {"beam_size": 2}, + "3i": {"beam_size": 2}, + "pi": {"beam_size": 256}, + "ip": {"beam_size": 256}, + "2u": {"beam_size": 512, "tnorm": "min"}, + "up": {"beam_size": 512} + } + }, + "reference": { + "notes": ["All 16 types are enabled, including the five negated types using CQD-A signed-atom negation with +H scoring.", "Released 3p/4p quirks are preserved."] + } + }, + { + "dataset": "ICEWS18+H", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/ICEWS18/checkpoint", + "options": { + "per_shape": { + "2p": {"beam_size": 32}, + "3p": {"beam_size": 2}, + "2i": {"beam_size": 2}, + "3i": {"beam_size": 2}, + "pi": {"beam_size": 512}, + "ip": {"beam_size": 256}, + "2u": {"beam_size": 2, "tnorm": "min"}, + "up": {"beam_size": 2} + } + }, + "reference": { + "notes": ["All 16 types are enabled, including the five negated types using CQD-A signed-atom negation with +H scoring.", "Released 3p/4p quirks are preserved."] + } + } + ] + }, + { + "defaults": { + "id": "cqd-hybrid-{dataset}", + "method": "cqd-hybrid", + "options": { + "beam_size": 2, + "tnorm": "prod", + "max_k": 512, + "normalize": true, + "reference_quirks": true, + "row_batch_size": 32, + "cache_bytes": 536870912, + "per_shape": { + "2p": {"tnorm": "prod"}, + "3p": {"tnorm": "prod"}, + "2i": {"tnorm": "prod"}, + "3i": {"tnorm": "prod"}, + "pi": {"tnorm": "prod"}, + "ip": {"tnorm": "prod"}, + "2u": {}, + "up": {"tnorm": "min"}, + "4p": {"beam_size": 2, "tnorm": "prod"}, + "4i": {"beam_size": 2, "tnorm": "prod"} + }, + "reference_batching": true, + "final_batch_size": 32, + "atomic_negation": true + }, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni", "4p", "4i"], + "query_batch_size": 1, + "query_order": "upstream-pickle", + "reference": {"status": "native reference port; full reproduction pending", "settings": "https://arxiv.org/html/2410.12537v3#A6"}, + "blockers": [], + "inference_graph": "train+valid" + }, + "entries": [ + { + "dataset": "FB15k237+H", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/fb15k237checkpoint", + "options": { + "per_shape": { + "2p": {"beam_size": 512}, + "3p": {"beam_size": 8}, + "2i": {"beam_size": 128}, + "3i": {"beam_size": 128}, + "pi": {"beam_size": 256}, + "ip": {"beam_size": 64}, + "2u": {"beam_size": 512, "tnorm": "prod"}, + "up": {"beam_size": 512} + } + }, + "reference": { + "notes": [ + "All 16 types are enabled, including the five negated types using CQD-A signed-atom negation with +H scoring.", + "Released 3p/4p quirks are preserved.", + "Stream the final projection with global stage normalization; bounded 4p reference preserves the released quirks.", + "FB 2u uses product: full-query CPU diagnostics reproduce published 40.1/42.2, while the appendix states min. The up setting uses the documented min." + ], + "inference_graph": "train+valid" + } + }, + { + "dataset": "NELL995+H", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/nellcheckpoint", + "options": { + "per_shape": { + "2p": {"beam_size": 512}, + "3p": {"beam_size": 2}, + "2i": {"beam_size": 2}, + "3i": {"beam_size": 2}, + "pi": {"beam_size": 256}, + "ip": {"beam_size": 256}, + "2u": {"beam_size": 512, "tnorm": "min"}, + "up": {"beam_size": 512} + } + }, + "reference": { + "notes": [ + "All 16 types are enabled, including the five negated types using CQD-A signed-atom negation with +H scoring.", + "Released 3p/4p quirks are preserved.", + "Stream the final projection with global stage normalization; bounded 4p reference preserves the released quirks." + ], + "inference_graph": "train+valid" + } + }, + { + "dataset": "ICEWS18+H", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/ICEWS18/checkpoint", + "options": { + "per_shape": { + "2p": {"beam_size": 32}, + "3p": {"beam_size": 2}, + "2i": {"beam_size": 2}, + "3i": {"beam_size": 2}, + "pi": {"beam_size": 512}, + "ip": {"beam_size": 256}, + "2u": {"beam_size": 2, "tnorm": "min"}, + "up": {"beam_size": 2}, + "4p": {"max_k": 64} + } + }, + "reference": { + "notes": [ + "All 16 types are enabled, including the five negated types using CQD-A signed-atom negation with +H scoring.", + "Released 3p/4p quirks are preserved.", + "Stream the final projection with global stage normalization; bounded 4p reference preserves the released quirks." + ], + "inference_graph": "unverified" + } + } + ] + }, + { + "defaults": { + "id": "qto-{dataset}", + "method": "qto", + "options": {"threshold": 0.0002, "row_batch_size": 32, "cache_bytes": 536870912, "reference_batching": true, "matrix_bytes": 6442450944, "host_matrix_bytes": 34359738368}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni", "4p", "4i"], + "query_batch_size": 1, + "query_order": "upstream-pickle", + "reference": { + "status": "native reference port; full reproduction pending", + "settings": "https://arxiv.org/html/2410.12537v3#A6", + "notes": [], + "inference_graph": "train" + }, + "blockers": [], + "inference_graph": "train+valid" + }, + "entries": [ + { + "dataset": "FB15k237+H", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/fb15k237checkpoint", + "options": {"negation_scale": 3} + }, + { + "dataset": "NELL995+H", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/nellcheckpoint", + "options": {"negation_scale": 6} + }, + { + "dataset": "ICEWS18+H", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/ICEWS18/checkpoint", + "options": {"negation_scale": 3} + } + ] + } + ] +} diff --git a/benchmarks/plus_h/kgfm_adapters.json b/benchmarks/plus_h/kgfm_adapters.json new file mode 100644 index 000000000..0d140d782 --- /dev/null +++ b/benchmarks/plus_h/kgfm_adapters.json @@ -0,0 +1,110 @@ +{ + "version": 1, + "answer_filter": "corrected", + "data_root": "KGs/query-benchmarks-plus-h", + "seed": 0, + "threads": 4, + "archives": { + "models": {"sha256": "d30b77a9c84d28de766bb5722c600bdc3c1e46d16cdaa3066ed4faca2b9b236f", "bytes": 4507216854, "source": "https://github.com/april-tools/is-cqa-complex#pre-trained-models"}, + "datasets": { + "source": "https://github.com/april-tools/is-cqa-complex/releases/tag/benchs-1.0", + "checksum_status": "Official release API digest; extracted runtime files are independently pinned by freeze.", + "sha256": "bf60135d7b1bc109ab9d90892e318fa96ace905fdc8204afd270b1fd98002a80", + "bytes": 1675873193 + } + }, + "recipes": [ + { + "defaults": { + "id": "ultra-product-intersections-{dataset}", + "method": "ultra-adapter", + "checkpoint": "checkpoints/ultra_3g.pth", + "adapters": {"product": "benchmarks/adapters/ultra_product_intersections.json"}, + "options": {"beam_size": 64, "row_batch_size": 8, "backend_batch_size": 8, "cache_bytes": 536870912, "raw_cache_bytes": 2147483648}, + "operators": { + "1p": "product", + "2p": "product", + "3p": "product", + "2i": "product", + "3i": "product", + "ip": "product", + "pi": "product", + "2u": "product", + "up": "product", + "2in": "product", + "3in": "product", + "inp": "product", + "pin": "product", + "pni": "product", + "4p": "product", + "4i": "product" + }, + "selection_protocol": "target-validation", + "finalized": true, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni", "4p", "4i"], + "query_batch_size": 16, + "query_order": "relation", + "reference": { + "status": "frozen shared 2-type recipe; verification pending", + "notes": [ + "Linear score-aware adapter, observed facts, bias bound 8, scale bound 2; trained on 2i and 3i source queries; product t-norm; k=64.", + "Training graphs: FB15k237, WN18RR and CoDExMedium; checkpoint selection uses reserved source validation. No target test labels select this recipe.", + "One shared recipe across both backbones and both suites, selected on validation queries of both suites; this is not target-untuned zero-shot selection.", + "The 4p and 4i types were not included in adapter training.", + "Paired learned/identity calibration: alpha=1 and beta=0 in the control; retain observed facts, normalization, query plan, operators and beam width. Share exact atomic batches through a bounded 2 GiB CPU cache; split the 512 MiB calibrated cache equally." + ] + }, + "blockers": [], + "adapter_ablation": true, + "inference_graph": "train+valid" + }, + "entries": [{"dataset": "FB15k237+H"}, {"dataset": "NELL995+H"}, {"dataset": "ICEWS18+H"}] + }, + { + "defaults": { + "id": "trix-product-intersections-{dataset}", + "method": "trix-adapter", + "checkpoint": "checkpoints/trix/entity_prediction.pth", + "adapters": {"product": "benchmarks/adapters/trix_product_intersections.json"}, + "options": {"beam_size": 64, "row_batch_size": 1, "backend_batch_size": 1, "cache_bytes": 536870912, "raw_cache_bytes": 2147483648}, + "operators": { + "1p": "product", + "2p": "product", + "3p": "product", + "2i": "product", + "3i": "product", + "ip": "product", + "pi": "product", + "2u": "product", + "up": "product", + "2in": "product", + "3in": "product", + "inp": "product", + "pin": "product", + "pni": "product", + "4p": "product", + "4i": "product" + }, + "selection_protocol": "target-validation", + "finalized": true, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni", "4p", "4i"], + "query_batch_size": 16, + "query_order": "relation", + "reference": { + "status": "frozen shared 2-type recipe; verification pending", + "notes": [ + "Linear score-aware adapter, observed facts, bias bound 8, scale bound 2; trained on 2i and 3i source queries; product t-norm; k=64.", + "Training graphs: FB15k237, WN18RR and CoDExMedium; checkpoint selection uses reserved source validation. No target test labels select this recipe.", + "One shared recipe across both backbones and both suites, selected on validation queries of both suites; this is not target-untuned zero-shot selection.", + "The 4p and 4i types were not included in adapter training.", + "Paired learned/identity calibration: alpha=1 and beta=0 in the control; retain observed facts, normalization, query plan, operators and beam width. Share exact atomic batches through a bounded 2 GiB CPU cache; split the 512 MiB calibrated cache equally." + ] + }, + "blockers": [], + "adapter_ablation": true, + "inference_graph": "train+valid" + }, + "entries": [{"dataset": "FB15k237+H"}, {"dataset": "NELL995+H"}, {"dataset": "ICEWS18+H"}] + } + ] +} diff --git a/benchmarks/plus_h/kgfm_kgicl.json b/benchmarks/plus_h/kgfm_kgicl.json new file mode 100644 index 000000000..ce1dbf968 --- /dev/null +++ b/benchmarks/plus_h/kgfm_kgicl.json @@ -0,0 +1,102 @@ +{ + "version": 1, + "answer_filter": "corrected", + "data_root": "KGs/query-benchmarks-plus-h", + "seed": 0, + "threads": 4, + "archives": { + "models": { + "sha256": "d30b77a9c84d28de766bb5722c600bdc3c1e46d16cdaa3066ed4faca2b9b236f", + "bytes": 4507216854, + "source": "https://github.com/april-tools/is-cqa-complex#pre-trained-models" + }, + "datasets": { + "source": "https://github.com/april-tools/is-cqa-complex/releases/tag/benchs-1.0", + "checksum_status": "Official release API digest; extracted runtime files are independently pinned by freeze.", + "sha256": "bf60135d7b1bc109ab9d90892e318fa96ace905fdc8204afd270b1fd98002a80", + "bytes": 1675873193 + } + }, + "recipes": [ + { + "defaults": { + "id": "kgicl-product-intersections-{dataset}", + "method": "kgicl-adapter", + "checkpoint": "checkpoints/kgicl/KG-ICL-6L/model_best.tar", + "adapters": { + "product": "benchmarks/adapters/kgicl_product_intersections.json" + }, + "options": { + "beam_size": 64, + "row_batch_size": 8, + "backend_batch_size": 8, + "cache_bytes": 536870912, + "raw_cache_bytes": 2147483648, + "projection_cache_mb": 512 + }, + "operators": { + "1p": "product", + "2p": "product", + "3p": "product", + "2i": "product", + "3i": "product", + "ip": "product", + "pi": "product", + "2u": "product", + "up": "product", + "2in": "product", + "3in": "product", + "inp": "product", + "pin": "product", + "pni": "product", + "4p": "product", + "4i": "product" + }, + "selection_protocol": "target-validation", + "finalized": true, + "query_types": [ + "1p", + "2p", + "3p", + "2i", + "3i", + "ip", + "pi", + "2u", + "up", + "2in", + "3in", + "inp", + "pin", + "pni", + "4p", + "4i" + ], + "query_batch_size": 16, + "query_order": "relation", + "reference": { + "status": "KG-ICL adapter recipe, fitted and pinned; additional backbone", + "notes": [ + "Adapter fitted with the shared ULTRA/TRIX recipe (linear score-aware adapter, observed facts, bias bound 8, scale bound 2, 2i/3i source queries of FB15k237, WN18RR and CoDExMedium, source-validation checkpoint selection; training seed 2026090851) and pinned before any test run.", + "Backbone: DICE KG-ICL with the official KG-ICL-6L weights (nju-websoft/KG-ICL 6a3166e3), verified against the pinned code plus tests/fixtures/kgicl/upstream-fixes.patch; see docs/kgicl.md.", + "Prompt graphs: five examples per relation, sampled from the attached inference graph with prompt seed 0, identical for every query batch and call order; no answer-distance mask. CQD beam 64, product t-norm, paired identity control. The per-layer relation tables of every query relation are cached (projection_cache_mb 512, about 260 MiB for FB15k237); with the default 64 MiB they are recomputed on graphs with many relations, about 2.5 times slower." + ] + }, + "blockers": [], + "adapter_ablation": true, + "inference_graph": "train+valid" + }, + "entries": [ + { + "dataset": "FB15k237+H" + }, + { + "dataset": "NELL995+H" + }, + { + "dataset": "ICEWS18+H" + } + ] + } + ] +} diff --git a/benchmarks/plus_h/published_results.json b/benchmarks/plus_h/published_results.json new file mode 100644 index 000000000..85fb31d9b --- /dev/null +++ b/benchmarks/plus_h/published_results.json @@ -0,0 +1,390 @@ +{ + "source": "https://github.com/april-tools/is-cqa-complex/blob/d1ce74164936a7c09d9147e83190da047cb39429/README.md", + "unit": "MRR x100; rounded published values", + "datasets": { + "FB15k237+H": { + "gnnqe": { + "1p": 42.8, + "2p": 5.2, + "3p": 4.0, + "2i": 6.0, + "3i": 8.8, + "pi": 5.6, + "ip": 9.9, + "2u": 32.5, + "up": 10.0, + "4p": 4.3, + "4i": 20.0, + "2in": 6.8, + "3in": 6.5, + "pin": 3.7, + "pni": 5.0, + "inp": 3.3 + }, + "ultraquery": { + "1p": 40.6, + "2p": 4.5, + "3p": 3.5, + "2i": 5.2, + "3i": 7.2, + "pi": 5.3, + "ip": 10.1, + "2u": 29.4, + "up": 8.3, + "4p": 3.8, + "4i": 16.4, + "2in": 5.3, + "3in": 5.5, + "pin": 2.6, + "pni": 3.7, + "inp": 2.2 + }, + "cqd": { + "1p": 46.7, + "2p": 4.4, + "3p": 2.4, + "2i": 11.3, + "3i": 12.8, + "pi": 6.0, + "ip": 11.5, + "2u": 40.1, + "up": 10.6, + "4p": 1.1, + "4i": 23.8, + "2in": 3.3, + "3in": 2.6, + "pin": 0.6, + "pni": 4.9, + "inp": 1.2 + }, + "cqd-hybrid": { + "1p": 46.7, + "2p": 4.8, + "3p": 3.1, + "2i": 6.0, + "3i": 8.6, + "pi": 5.5, + "ip": 12.9, + "2u": 42.2, + "up": 12.0, + "4p": 2.4, + "4i": 17.4, + "2in": 4.7, + "3in": 1.6, + "pin": 1.0, + "pni": 3.2, + "inp": 1.3 + }, + "cone": { + "1p": 41.8, + "2p": 4.6, + "3p": 3.9, + "2i": 9.1, + "3i": 10.3, + "pi": 3.8, + "ip": 7.9, + "2u": 22.8, + "up": 6.0, + "4p": 3.5, + "4i": 20.3, + "2in": 5.1, + "3in": 4.9, + "pin": 2.9, + "pni": 3.3, + "inp": 3.6 + }, + "qto": { + "1p": 46.7, + "2p": 4.9, + "3p": 3.7, + "2i": 8.7, + "3i": 10.1, + "pi": 6.1, + "ip": 13.5, + "2u": 30.6, + "up": 11.2, + "4p": 3.9, + "4i": 20.2, + "2in": 10.6, + "3in": 3.1, + "pin": 2.0, + "pni": 5.3, + "inp": 1.5 + }, + "clmpt": { + "1p": 45.3, + "2p": 5.3, + "3p": 4.7, + "2i": 10.2, + "3i": 12.2, + "pi": 5.6, + "ip": 14.9, + "2u": 33.6, + "up": 14.2, + "4p": 4.5, + "4i": 24.0, + "2in": 6.8, + "3in": 2.3, + "pin": 1.6, + "pni": 4.8, + "inp": 2.5 + } + }, + "NELL995+H": { + "gnnqe": { + "1p": 53.6, + "2p": 8.0, + "3p": 6.0, + "2i": 10.7, + "3i": 13.3, + "pi": 16.0, + "ip": 13.5, + "2u": 47.5, + "up": 9.8, + "4p": 4.7, + "4i": 19.4, + "2in": 5.5, + "3in": 6.4, + "pin": 5.8, + "pni": 3.3, + "inp": 4.4 + }, + "ultraquery": { + "1p": 38.9, + "2p": 6.1, + "3p": 4.1, + "2i": 7.9, + "3i": 10.2, + "pi": 15.8, + "ip": 9.3, + "2u": 28.1, + "up": 9.5, + "4p": 4.2, + "4i": 15.6, + "2in": 4.5, + "3in": 5.9, + "pin": 4.3, + "pni": 2.7, + "inp": 3.6 + }, + "cqd": { + "1p": 60.4, + "2p": 9.6, + "3p": 4.2, + "2i": 18.5, + "3i": 19.6, + "pi": 18.9, + "ip": 22.6, + "2u": 46.3, + "up": 18.5, + "4p": 2.0, + "4i": 24.8, + "2in": 4.2, + "3in": 1.5, + "pin": 1.5, + "pni": 4.9, + "inp": 2.6 + }, + "cqd-hybrid": { + "1p": 60.4, + "2p": 9.0, + "3p": 6.1, + "2i": 12.1, + "3i": 14.4, + "pi": 17.4, + "ip": 21.2, + "2u": 46.4, + "up": 19.3, + "4p": 3.5, + "4i": 20.4, + "2in": 5.1, + "3in": 1.2, + "pin": 1.4, + "pni": 4.3, + "inp": 2.4 + }, + "cone": { + "1p": 53.1, + "2p": 7.9, + "3p": 6.7, + "2i": 21.8, + "3i": 23.6, + "pi": 14.9, + "ip": 11.8, + "2u": 39.9, + "up": 8.8, + "4p": 5.2, + "4i": 27.6, + "2in": 4.6, + "3in": 6.0, + "pin": 3.7, + "pni": 2.7, + "inp": 6.4 + }, + "qto": { + "1p": 60.3, + "2p": 9.8, + "3p": 8.0, + "2i": 14.6, + "3i": 15.8, + "pi": 17.6, + "ip": 21.1, + "2u": 49.1, + "up": 18.9, + "4p": 7.0, + "4i": 20.9, + "2in": 10.2, + "3in": 2.3, + "pin": 3.1, + "pni": 8.4, + "inp": 2.4 + }, + "clmpt": { + "1p": 58.1, + "2p": 10.1, + "3p": 7.8, + "2i": 22.7, + "3i": 25.0, + "pi": 17.2, + "ip": 24.4, + "2u": 50.0, + "up": 22.0, + "4p": 7.2, + "4i": 29.1, + "2in": 6.5, + "3in": 2.4, + "pin": 4.1, + "pni": 2.3, + "inp": 4.5 + } + }, + "ICEWS18+H": { + "gnnqe": { + "1p": 12.2, + "2p": 0.9, + "3p": 0.5, + "2i": 16.1, + "3i": 26.5, + "pi": 19.1, + "ip": 3.5, + "2u": 7.6, + "up": 1.1, + "4p": 0.4, + "4i": 34.0, + "2in": 4.5, + "3in": 6.9, + "pin": 0.9, + "pni": 3.5, + "inp": 0.8 + }, + "ultraquery": { + "1p": 6.3, + "2p": 1.2, + "3p": 1.2, + "2i": 7.0, + "3i": 11.7, + "pi": 8.8, + "ip": 1.3, + "2u": 3.3, + "up": 1.2, + "4p": 0.8, + "4i": 15.9, + "2in": 2.3, + "3in": 4.8, + "pin": 1.2, + "pni": 2.2, + "inp": 1.6 + }, + "cqd": { + "1p": 16.6, + "2p": 2.5, + "3p": 1.5, + "2i": 13.0, + "3i": 19.5, + "pi": 17.1, + "ip": 6.7, + "2u": 6.8, + "up": 5.9, + "4p": 1.1, + "4i": 24.0, + "2in": 1.5, + "3in": 2.9, + "pin": 0.2, + "pni": 2.7, + "inp": 0.9 + }, + "cqd-hybrid": { + "1p": 16.6, + "2p": 2.6, + "3p": 1.5, + "2i": 15.0, + "3i": 25.5, + "pi": 17.5, + "ip": 5.8, + "2u": 6.8, + "up": 5.6, + "4p": 0.9, + "4i": 33.2, + "2in": 1.7, + "3in": 4.0, + "pin": 0.3, + "pni": 2.2, + "inp": 1.1 + }, + "cone": { + "1p": 3.5, + "2p": 0.9, + "3p": 0.9, + "2i": 1.2, + "3i": 0.5, + "pi": 1.2, + "ip": 1.6, + "2u": 1.1, + "up": 0.9, + "4p": 0.6, + "4i": 0.3, + "2in": 1.7, + "3in": 2.9, + "pin": 1.1, + "pni": 0.9, + "inp": 1.3 + }, + "qto": { + "1p": 16.6, + "2p": 2.6, + "3p": 1.4, + "2i": 15.7, + "3i": 25.0, + "pi": 18.4, + "ip": 6.2, + "2u": 6.7, + "up": 4.9, + "4p": 1.1, + "4i": 31.5, + "2in": 4.9, + "3in": 8.7, + "pin": 1.2, + "pni": 3.0, + "inp": 0.9 + }, + "clmpt": { + "1p": 4.7, + "2p": 0.8, + "3p": 0.1, + "2i": 12.0, + "3i": 23.0, + "pi": 9.7, + "ip": 2.1, + "2u": 2.7, + "up": 2.2, + "4p": 0.1, + "4i": 31.0, + "2in": 1.2, + "3in": 2.1, + "pin": 0.1, + "pni": 1.0, + "inp": 0.2 + } + } + } +} diff --git a/benchmarks/ultraquery/baselines.json b/benchmarks/ultraquery/baselines.json new file mode 100644 index 000000000..b325a72d3 --- /dev/null +++ b/benchmarks/ultraquery/baselines.json @@ -0,0 +1,55 @@ +{ + "version": 1, + "answer_filter": "released", + "data_root": "KGs/UltraQuery", + "seed": 0, + "threads": 4, + "archives": { + "datasets": {"source": "https://github.com/DeepGraphLearning/ULTRA/tree/427966ad8ed60420eef034063d44f3153addff90#datasets-1", "checksum_status": "Every extracted runtime input is independently pinned by freeze."}, + "models": {"source": "https://github.com/DeepGraphLearning/ULTRA/tree/427966ad8ed60420eef034063d44f3153addff90/ckpts", "checksum_status": "Released checkpoint and adapter bytes are independently pinned by freeze."} + }, + "recipes": [ + { + "defaults": { + "id": "ultraquery-{dataset}", + "method": "ultraquery", + "checkpoint": "Experiments/query-baselines/upstream/ultra/ckpts/ultraquery.pth", + "options": {"logic": "product", "threshold": 0.0, "cache_bytes": 67108864}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni"], + "query_batch_size": 32, + "query_order": "upstream-pickle", + "reference": { + "status": "released PyG port; frozen validation parity required", + "settings": "https://github.com/DeepGraphLearning/ULTRA/blob/427966ad8ed60420eef034063d44f3153addff90/README.md#ultraquery", + "notes": ["Public query-trained checkpoint, product logic, threshold 0.0, published batch size 32.", "Released PyG implementation; original paper used a different TorchDrug checkpoint."] + }, + "blockers": [] + }, + "entries": [ + {"dataset": "FB15k237LogicalQuery"}, + {"dataset": "FB15kLogicalQuery"}, + {"dataset": "NELL995LogicalQuery"}, + {"dataset": "InductiveFB15k237Query:106"}, + {"dataset": "InductiveFB15k237Query:113"}, + {"dataset": "InductiveFB15k237Query:122"}, + {"dataset": "InductiveFB15k237Query:134"}, + {"dataset": "InductiveFB15k237Query:150"}, + {"dataset": "InductiveFB15k237Query:175"}, + {"dataset": "InductiveFB15k237Query:217"}, + {"dataset": "InductiveFB15k237Query:300"}, + {"dataset": "InductiveFB15k237Query:550"}, + {"dataset": "WikiTopicsQuery:art"}, + {"dataset": "WikiTopicsQuery:award"}, + {"dataset": "WikiTopicsQuery:edu"}, + {"dataset": "WikiTopicsQuery:health"}, + {"dataset": "WikiTopicsQuery:infra"}, + {"dataset": "WikiTopicsQuery:loc"}, + {"dataset": "WikiTopicsQuery:org"}, + {"dataset": "WikiTopicsQuery:people"}, + {"dataset": "WikiTopicsQuery:sci"}, + {"dataset": "WikiTopicsQuery:sport"}, + {"dataset": "WikiTopicsQuery:tax"} + ] + } + ] +} diff --git a/benchmarks/ultraquery/comparisons.json b/benchmarks/ultraquery/comparisons.json new file mode 100644 index 000000000..a435a054a --- /dev/null +++ b/benchmarks/ultraquery/comparisons.json @@ -0,0 +1,229 @@ +{ + "version": 1, + "answer_filter": "released", + "data_root": "KGs/UltraQuery", + "seed": 0, + "threads": 4, + "archives": { + "datasets": {"source": "https://github.com/DeepGraphLearning/ULTRA/tree/427966ad8ed60420eef034063d44f3153addff90#datasets-1", "checksum_status": "Every extracted runtime input is independently pinned by freeze."}, + "models": {"source": "https://github.com/DeepGraphLearning/ULTRA/tree/427966ad8ed60420eef034063d44f3153addff90/ckpts", "checksum_status": "Released checkpoint and adapter bytes are independently pinned by freeze."} + }, + "recipes": [ + { + "defaults": { + "id": "ultraquery-lp-{dataset}", + "method": "ultraquery-lp", + "checkpoint": "Experiments/query-baselines/upstream/ultra/ckpts/ultra_3g.pth", + "options": {"logic": "product", "cache_bytes": 67108864}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni"], + "query_batch_size": 32, + "query_order": "upstream-pickle", + "reference": { + "status": "released PyG port; frozen validation parity required", + "settings": "https://arxiv.org/html/2404.07198v2#S0.SS2", + "notes": ["Released PyG 3g link-prediction checkpoint; thresholds follow UltraQuery arXiv v2, Appendix B.", "Released PyG checkpoint is distinct from original TorchDrug paper weights."] + }, + "blockers": [] + }, + "entries": [ + {"dataset": "FB15k237LogicalQuery", "options": {"threshold": 0.8}}, + { + "dataset": "FB15k237LogicalQuery", + "id": "ultraquery-lp-no-threshold-FB15k237LogicalQuery", + "options": {"threshold": 0.0} + }, + {"dataset": "FB15kLogicalQuery", "options": {"threshold": 0.8}}, + { + "dataset": "FB15kLogicalQuery", + "id": "ultraquery-lp-no-threshold-FB15kLogicalQuery", + "options": {"threshold": 0.0} + }, + {"dataset": "NELL995LogicalQuery", "options": {"threshold": 0.97}}, + { + "dataset": "NELL995LogicalQuery", + "id": "ultraquery-lp-no-threshold-NELL995LogicalQuery", + "options": {"threshold": 0.0} + }, + {"dataset": "InductiveFB15k237Query:106", "options": {"threshold": 0.8}}, + { + "dataset": "InductiveFB15k237Query:106", + "id": "ultraquery-lp-no-threshold-InductiveFB15k237Query:106", + "options": {"threshold": 0.0} + }, + {"dataset": "InductiveFB15k237Query:113", "options": {"threshold": 0.8}}, + { + "dataset": "InductiveFB15k237Query:113", + "id": "ultraquery-lp-no-threshold-InductiveFB15k237Query:113", + "options": {"threshold": 0.0} + }, + {"dataset": "InductiveFB15k237Query:122", "options": {"threshold": 0.8}}, + { + "dataset": "InductiveFB15k237Query:122", + "id": "ultraquery-lp-no-threshold-InductiveFB15k237Query:122", + "options": {"threshold": 0.0} + }, + {"dataset": "InductiveFB15k237Query:134", "options": {"threshold": 0.8}}, + { + "dataset": "InductiveFB15k237Query:134", + "id": "ultraquery-lp-no-threshold-InductiveFB15k237Query:134", + "options": {"threshold": 0.0} + }, + {"dataset": "InductiveFB15k237Query:150", "options": {"threshold": 0.8}}, + { + "dataset": "InductiveFB15k237Query:150", + "id": "ultraquery-lp-no-threshold-InductiveFB15k237Query:150", + "options": {"threshold": 0.0} + }, + {"dataset": "InductiveFB15k237Query:175", "options": {"threshold": 0.8}}, + { + "dataset": "InductiveFB15k237Query:175", + "id": "ultraquery-lp-no-threshold-InductiveFB15k237Query:175", + "options": {"threshold": 0.0} + }, + {"dataset": "InductiveFB15k237Query:217", "options": {"threshold": 0.8}}, + { + "dataset": "InductiveFB15k237Query:217", + "id": "ultraquery-lp-no-threshold-InductiveFB15k237Query:217", + "options": {"threshold": 0.0} + }, + {"dataset": "InductiveFB15k237Query:300", "options": {"threshold": 0.8}}, + { + "dataset": "InductiveFB15k237Query:300", + "id": "ultraquery-lp-no-threshold-InductiveFB15k237Query:300", + "options": {"threshold": 0.0} + }, + {"dataset": "InductiveFB15k237Query:550", "options": {"threshold": 0.8}}, + { + "dataset": "InductiveFB15k237Query:550", + "id": "ultraquery-lp-no-threshold-InductiveFB15k237Query:550", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:art", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:art", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:art", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:award", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:award", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:award", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:edu", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:edu", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:edu", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:health", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:health", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:health", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:infra", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:infra", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:infra", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:loc", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:loc", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:loc", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:org", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:org", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:org", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:people", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:people", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:people", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:sci", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:sci", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:sci", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:sport", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:sport", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:sport", + "options": {"threshold": 0.0} + }, + {"dataset": "WikiTopicsQuery:tax", "options": {"threshold": 0.8}}, + { + "dataset": "WikiTopicsQuery:tax", + "id": "ultraquery-lp-no-threshold-WikiTopicsQuery:tax", + "options": {"threshold": 0.0} + } + ] + }, + { + "defaults": { + "id": "incoming-relation-{dataset}", + "method": "incoming-relation", + "checkpoint": "benchmarks/ultraquery/heuristic.json", + "options": {}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni"], + "query_batch_size": 32, + "query_order": "upstream-pickle", + "reference": { + "status": "untrained authors heuristic; independent validation parity required", + "settings": "https://github.com/DeepGraphLearning/InductiveQE/tree/ab49a0e64e3449e687e484a42269d9898e1d1325", + "notes": ["Boolean incoming-relation classes; query-seeded random shuffle for resumable evaluation."] + }, + "blockers": [] + }, + "entries": [ + {"dataset": "WikiTopicsQuery:art"}, + {"dataset": "WikiTopicsQuery:award"}, + {"dataset": "WikiTopicsQuery:edu"}, + {"dataset": "WikiTopicsQuery:health"}, + {"dataset": "WikiTopicsQuery:infra"}, + {"dataset": "WikiTopicsQuery:loc"}, + {"dataset": "WikiTopicsQuery:org"}, + {"dataset": "WikiTopicsQuery:people"}, + {"dataset": "WikiTopicsQuery:sci"}, + {"dataset": "WikiTopicsQuery:sport"}, + {"dataset": "WikiTopicsQuery:tax"} + ] + }, + { + "defaults": { + "id": "qto-{dataset}", + "method": "qto", + "options": {"threshold": 0.0002, "row_batch_size": 32, "cache_bytes": 536870912, "reference_batching": true, "matrix_bytes": 6442450944, "host_matrix_bytes": 34359738368}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni"], + "query_batch_size": 1, + "query_order": "upstream-pickle", + "reference": { + "status": "native reference port; full reproduction pending", + "settings": "https://arxiv.org/html/2410.12537v3#A6", + "notes": ["Reuse +H released weights: original ID maps and training triples are identical; verify against QTO on original query sets."], + "inference_graph": "train" + }, + "blockers": [] + }, + "entries": [ + { + "dataset": "FB15k237LogicalQuery", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/fb15k237checkpoint", + "options": {"negation_scale": 3} + }, + { + "dataset": "NELL995LogicalQuery", + "checkpoint": "checkpoints/query-baselines/iscqa-compl-models/models/CQD:CQD-HYBRID:QTO/QTO/nellcheckpoint", + "options": {"negation_scale": 6} + } + ] + } + ] +} diff --git a/benchmarks/ultraquery/heuristic.json b/benchmarks/ultraquery/heuristic.json new file mode 100644 index 000000000..8ef063593 --- /dev/null +++ b/benchmarks/ultraquery/heuristic.json @@ -0,0 +1,6 @@ +{ + "version": 1, + "method": "incoming-relation", + "reference_commit": "ab49a0e64e3449e687e484a42269d9898e1d1325", + "ranking": "query-seeded random shuffle within Boolean classes" +} diff --git a/benchmarks/ultraquery/inductive-v2-files.json b/benchmarks/ultraquery/inductive-v2-files.json new file mode 100644 index 000000000..28dd2c8f1 --- /dev/null +++ b/benchmarks/ultraquery/inductive-v2-files.json @@ -0,0 +1,636 @@ +{ + "version": "2.0", + "source": "https://zenodo.org/records/7306046", + "files": { + "106": { + "train_answers_valid.pkl": { + "bytes": 222615050, + "crc32": 2127457251 + }, + "val_predict.txt": { + "bytes": 146463, + "crc32": 1962570836 + }, + "test_answers_hard.pkl": { + "bytes": 8012856, + "crc32": 4274819325 + }, + "valid_answers_easy.pkl": { + "bytes": 190930802, + "crc32": 2032002918 + }, + "stats.txt": { + "bytes": 110, + "crc32": 916940677 + }, + "valid_answers_hard.pkl": { + "bytes": 9337662, + "crc32": 2641552673 + }, + "test_queries.pkl": { + "bytes": 3904572, + "crc32": 3909217021 + }, + "train_queries.pkl": { + "bytes": 33327290, + "crc32": 170044298 + }, + "og_mappings.pkl": { + "bytes": 351138, + "crc32": 614139134 + }, + "val_inference.txt": { + "bytes": 829929, + "crc32": 3500356919 + }, + "train_answers_test.pkl": { + "bytes": 221922191, + "crc32": 61633874 + }, + "test_predict.txt": { + "bytes": 114275, + "crc32": 515135815 + }, + "train_answers_hard.pkl": { + "bytes": 256358832, + "crc32": 537753088 + }, + "test_inference.txt": { + "bytes": 647847, + "crc32": 3601583391 + }, + "test_answers_easy.pkl": { + "bytes": 195359464, + "crc32": 2584167393 + }, + "train_graph.txt": { + "bytes": 6397355, + "crc32": 956701669 + }, + "valid_queries.pkl": { + "bytes": 3928336, + "crc32": 1356027438 + } + }, + "113": { + "train_answers_valid.pkl": { + "bytes": 225603284, + "crc32": 4279546648 + }, + "val_predict.txt": { + "bytes": 225526, + "crc32": 2153236401 + }, + "test_answers_hard.pkl": { + "bytes": 10082998, + "crc32": 2283473921 + }, + "valid_answers_easy.pkl": { + "bytes": 157677730, + "crc32": 4145877345 + }, + "stats.txt": { + "bytes": 112, + "crc32": 4251385914 + }, + "valid_answers_hard.pkl": { + "bytes": 11342559, + "crc32": 415546472 + }, + "test_queries.pkl": { + "bytes": 3981855, + "crc32": 903912400 + }, + "train_queries.pkl": { + "bytes": 32966406, + "crc32": 1582195831 + }, + "og_mappings.pkl": { + "bytes": 350013, + "crc32": 1962790053 + }, + "val_inference.txt": { + "bytes": 1277765, + "crc32": 1428619069 + }, + "train_answers_test.pkl": { + "bytes": 218874594, + "crc32": 991468924 + }, + "test_predict.txt": { + "bytes": 211532, + "crc32": 2320895943 + }, + "train_answers_hard.pkl": { + "bytes": 239712017, + "crc32": 2821696352 + }, + "test_inference.txt": { + "bytes": 1198069, + "crc32": 2029575231 + }, + "test_answers_easy.pkl": { + "bytes": 153584506, + "crc32": 2208956229 + }, + "train_graph.txt": { + "bytes": 5141002, + "crc32": 89518030 + }, + "valid_queries.pkl": { + "bytes": 3989231, + "crc32": 4285792803 + } + }, + "122": { + "train_answers_valid.pkl": { + "bytes": 227775955, + "crc32": 1925017448 + }, + "val_predict.txt": { + "bytes": 288572, + "crc32": 427082963 + }, + "test_answers_hard.pkl": { + "bytes": 17676225, + "crc32": 1069461696 + }, + "valid_answers_easy.pkl": { + "bytes": 166736811, + "crc32": 3894232451 + }, + "stats.txt": { + "bytes": 112, + "crc32": 4038247536 + }, + "valid_answers_hard.pkl": { + "bytes": 15165545, + "crc32": 4175911720 + }, + "test_queries.pkl": { + "bytes": 5128445, + "crc32": 2878735352 + }, + "train_queries.pkl": { + "bytes": 32535566, + "crc32": 3346475155 + }, + "og_mappings.pkl": { + "bytes": 349725, + "crc32": 1969171057 + }, + "val_inference.txt": { + "bytes": 1635657, + "crc32": 805202829 + }, + "train_answers_test.pkl": { + "bytes": 227079621, + "crc32": 1548910851 + }, + "test_predict.txt": { + "bytes": 333248, + "crc32": 3875030041 + }, + "train_answers_hard.pkl": { + "bytes": 226863249, + "crc32": 3565487833 + }, + "test_inference.txt": { + "bytes": 1886648, + "crc32": 1080774119 + }, + "test_answers_easy.pkl": { + "bytes": 166213516, + "crc32": 3869019787 + }, + "train_graph.txt": { + "bytes": 3816444, + "crc32": 146621670 + }, + "valid_queries.pkl": { + "bytes": 5089342, + "crc32": 113618264 + } + }, + "134": { + "train_answers_valid.pkl": { + "bytes": 232388780, + "crc32": 638433923 + }, + "val_predict.txt": { + "bytes": 340343, + "crc32": 1546527536 + }, + "test_answers_hard.pkl": { + "bytes": 45573463, + "crc32": 3916559349 + }, + "valid_answers_easy.pkl": { + "bytes": 327167263, + "crc32": 132510776 + }, + "stats.txt": { + "bytes": 111, + "crc32": 3490429564 + }, + "valid_answers_hard.pkl": { + "bytes": 44755981, + "crc32": 1021389800 + }, + "test_queries.pkl": { + "bytes": 15812052, + "crc32": 1569178564 + }, + "train_queries.pkl": { + "bytes": 32230339, + "crc32": 1812113743 + }, + "og_mappings.pkl": { + "bytes": 346933, + "crc32": 4124865534 + }, + "val_inference.txt": { + "bytes": 1932846, + "crc32": 3024773898 + }, + "train_answers_test.pkl": { + "bytes": 232194115, + "crc32": 1820189611 + }, + "test_predict.txt": { + "bytes": 348401, + "crc32": 1832583827 + }, + "train_answers_hard.pkl": { + "bytes": 216024902, + "crc32": 1503661660 + }, + "test_inference.txt": { + "bytes": 1977504, + "crc32": 903245296 + }, + "test_answers_easy.pkl": { + "bytes": 334834974, + "crc32": 3316223477 + }, + "train_graph.txt": { + "bytes": 2884883, + "crc32": 2461824179 + }, + "valid_queries.pkl": { + "bytes": 15823689, + "crc32": 4154145113 + } + }, + "150": { + "train_answers_valid.pkl": { + "bytes": 206829540, + "crc32": 2622602600 + }, + "val_predict.txt": { + "bytes": 345019, + "crc32": 374871620 + }, + "test_answers_hard.pkl": { + "bytes": 72097527, + "crc32": 3921641806 + }, + "valid_answers_easy.pkl": { + "bytes": 429105480, + "crc32": 1305220952 + }, + "stats.txt": { + "bytes": 111, + "crc32": 1969255981 + }, + "valid_answers_hard.pkl": { + "bytes": 59967681, + "crc32": 3458923041 + }, + "test_queries.pkl": { + "bytes": 19385640, + "crc32": 2409607166 + }, + "train_queries.pkl": { + "bytes": 31225181, + "crc32": 380877425 + }, + "og_mappings.pkl": { + "bytes": 347176, + "crc32": 2345290980 + }, + "val_inference.txt": { + "bytes": 1956323, + "crc32": 2462227954 + }, + "train_answers_test.pkl": { + "bytes": 222962542, + "crc32": 655110866 + }, + "test_predict.txt": { + "bytes": 425310, + "crc32": 457502557 + }, + "train_answers_hard.pkl": { + "bytes": 177217090, + "crc32": 774506850 + }, + "test_inference.txt": { + "bytes": 2412571, + "crc32": 545194768 + }, + "test_answers_easy.pkl": { + "bytes": 472301691, + "crc32": 575961351 + }, + "train_graph.txt": { + "bytes": 2045704, + "crc32": 1094433206 + }, + "valid_queries.pkl": { + "bytes": 19336144, + "crc32": 3799685740 + } + }, + "175": { + "train_answers_valid.pkl": { + "bytes": 231658132, + "crc32": 2338021379 + }, + "val_predict.txt": { + "bytes": 374452, + "crc32": 3645624910 + }, + "test_answers_hard.pkl": { + "bytes": 70174732, + "crc32": 1097366851 + }, + "valid_answers_easy.pkl": { + "bytes": 309746882, + "crc32": 263560611 + }, + "stats.txt": { + "bytes": 111, + "crc32": 1426340539 + }, + "valid_answers_hard.pkl": { + "bytes": 62065943, + "crc32": 3481264274 + }, + "test_queries.pkl": { + "bytes": 20586045, + "crc32": 2267890764 + }, + "train_queries.pkl": { + "bytes": 30744012, + "crc32": 1428388548 + }, + "og_mappings.pkl": { + "bytes": 337664, + "crc32": 3639673031 + }, + "val_inference.txt": { + "bytes": 2125466, + "crc32": 245359974 + }, + "train_answers_test.pkl": { + "bytes": 237446394, + "crc32": 2418683699 + }, + "test_predict.txt": { + "bytes": 408845, + "crc32": 970098317 + }, + "train_answers_hard.pkl": { + "bytes": 170942744, + "crc32": 3838087073 + }, + "test_inference.txt": { + "bytes": 2320451, + "crc32": 1307371129 + }, + "test_answers_easy.pkl": { + "bytes": 341969689, + "crc32": 2430511246 + }, + "train_graph.txt": { + "bytes": 1268480, + "crc32": 3026052970 + }, + "valid_queries.pkl": { + "bytes": 20594963, + "crc32": 3394616982 + } + }, + "217": { + "train_answers_valid.pkl": { + "bytes": 136495499, + "crc32": 3465188741 + }, + "val_predict.txt": { + "bytes": 403705, + "crc32": 87815284 + }, + "test_answers_hard.pkl": { + "bytes": 65584728, + "crc32": 1167581732 + }, + "valid_answers_easy.pkl": { + "bytes": 221899548, + "crc32": 3501421439 + }, + "stats.txt": { + "bytes": 111, + "crc32": 3317629882 + }, + "valid_answers_hard.pkl": { + "bytes": 70989244, + "crc32": 39487799 + }, + "test_queries.pkl": { + "bytes": 20618039, + "crc32": 849975777 + }, + "train_queries.pkl": { + "bytes": 28096983, + "crc32": 1210595688 + }, + "og_mappings.pkl": { + "bytes": 338157, + "crc32": 2367516723 + }, + "val_inference.txt": { + "bytes": 2290131, + "crc32": 3491177888 + }, + "train_answers_test.pkl": { + "bytes": 141649548, + "crc32": 2003735472 + }, + "test_predict.txt": { + "bytes": 389882, + "crc32": 1434760274 + }, + "train_answers_hard.pkl": { + "bytes": 94947311, + "crc32": 3826735907 + }, + "test_inference.txt": { + "bytes": 2212932, + "crc32": 349255903 + }, + "test_answers_easy.pkl": { + "bytes": 219221355, + "crc32": 2211991067 + }, + "train_graph.txt": { + "bytes": 640360, + "crc32": 2677427181 + }, + "valid_queries.pkl": { + "bytes": 20668742, + "crc32": 1513051335 + } + }, + "300": { + "train_answers_valid.pkl": { + "bytes": 211853710, + "crc32": 1083432408 + }, + "val_predict.txt": { + "bytes": 353491, + "crc32": 2721905548 + }, + "test_answers_hard.pkl": { + "bytes": 75406827, + "crc32": 275037086 + }, + "valid_answers_easy.pkl": { + "bytes": 205819376, + "crc32": 2104015365 + }, + "stats.txt": { + "bytes": 111, + "crc32": 1757620724 + }, + "valid_answers_hard.pkl": { + "bytes": 64180660, + "crc32": 2866994750 + }, + "test_queries.pkl": { + "bytes": 20550283, + "crc32": 1154687234 + }, + "train_queries.pkl": { + "bytes": 24113843, + "crc32": 2428668404 + }, + "og_mappings.pkl": { + "bytes": 332839, + "crc32": 1225606516 + }, + "val_inference.txt": { + "bytes": 2009632, + "crc32": 1759567357 + }, + "train_answers_test.pkl": { + "bytes": 224335036, + "crc32": 364211603 + }, + "test_predict.txt": { + "bytes": 391694, + "crc32": 490777807 + }, + "train_answers_hard.pkl": { + "bytes": 102976653, + "crc32": 2066889623 + }, + "test_inference.txt": { + "bytes": 2225314, + "crc32": 1460261459 + }, + "test_answers_easy.pkl": { + "bytes": 258478502, + "crc32": 1161846747 + }, + "train_graph.txt": { + "bytes": 288179, + "crc32": 1711314424 + }, + "valid_queries.pkl": { + "bytes": 20523744, + "crc32": 2538452426 + } + }, + "550": { + "train_answers_valid.pkl": { + "bytes": 67939823, + "crc32": 1612544607 + }, + "val_predict.txt": { + "bytes": 300357, + "crc32": 964660324 + }, + "test_answers_hard.pkl": { + "bytes": 64205685, + "crc32": 2309798725 + }, + "valid_answers_easy.pkl": { + "bytes": 138521541, + "crc32": 2605770060 + }, + "stats.txt": { + "bytes": 111, + "crc32": 109310899 + }, + "valid_answers_hard.pkl": { + "bytes": 57947005, + "crc32": 2281276267 + }, + "test_queries.pkl": { + "bytes": 20293809, + "crc32": 816724508 + }, + "train_queries.pkl": { + "bytes": 20499571, + "crc32": 630566200 + }, + "og_mappings.pkl": { + "bytes": 314875, + "crc32": 298538 + }, + "val_inference.txt": { + "bytes": 1716884, + "crc32": 335509201 + }, + "train_answers_test.pkl": { + "bytes": 75661505, + "crc32": 3447717392 + }, + "test_predict.txt": { + "bytes": 305168, + "crc32": 1714620031 + }, + "train_answers_hard.pkl": { + "bytes": 39959829, + "crc32": 412282031 + }, + "test_inference.txt": { + "bytes": 1742124, + "crc32": 362131730 + }, + "test_answers_easy.pkl": { + "bytes": 174113211, + "crc32": 2351455741 + }, + "train_graph.txt": { + "bytes": 57819, + "crc32": 3024727647 + }, + "valid_queries.pkl": { + "bytes": 20303298, + "crc32": 820371586 + } + } + } +} diff --git a/benchmarks/ultraquery/kgfm_adapters.json b/benchmarks/ultraquery/kgfm_adapters.json new file mode 100644 index 000000000..115926223 --- /dev/null +++ b/benchmarks/ultraquery/kgfm_adapters.json @@ -0,0 +1,147 @@ +{ + "version": 1, + "answer_filter": "released", + "data_root": "KGs/UltraQuery", + "seed": 0, + "threads": 4, + "archives": { + "datasets": {"source": "https://github.com/DeepGraphLearning/ULTRA/tree/427966ad8ed60420eef034063d44f3153addff90#datasets-1", "checksum_status": "Every extracted runtime input is independently pinned by freeze."}, + "models": {"source": "https://github.com/DeepGraphLearning/ULTRA/tree/427966ad8ed60420eef034063d44f3153addff90/ckpts", "checksum_status": "Released checkpoint and adapter bytes are independently pinned by freeze."} + }, + "recipes": [ + { + "defaults": { + "id": "ultra-product-intersections-{dataset}", + "method": "ultra-adapter", + "checkpoint": "checkpoints/ultra_3g.pth", + "adapters": {"product": "benchmarks/adapters/ultra_product_intersections.json"}, + "options": {"beam_size": 64, "row_batch_size": 8, "backend_batch_size": 8, "cache_bytes": 536870912, "raw_cache_bytes": 2147483648}, + "operators": { + "1p": "product", + "2p": "product", + "3p": "product", + "2i": "product", + "3i": "product", + "ip": "product", + "pi": "product", + "2u": "product", + "up": "product", + "2in": "product", + "3in": "product", + "inp": "product", + "pin": "product", + "pni": "product" + }, + "selection_protocol": "target-validation", + "finalized": true, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni"], + "query_batch_size": 16, + "query_order": "relation", + "reference": { + "status": "frozen transfer recipe; independent KGFM verification required", + "notes": [ + "Frozen source-trained product adapter; CQD beam 64; paired learned/identity calibration with shared atomic batches.", + "Training graphs: FB15k237, WN18RR and CoDExMedium. No target test labels select these recipes.", + "Backbone parity is checked separately; per-run KGFM evidence verifies adapter/composition against independently captured atomic logits.", + "Adapter training uses only 2i and 3i; checkpoint selection uses reserved source validation.", + "One shared recipe across both backbones and both suites, selected on validation queries of both suites; this is not target-untuned zero-shot selection." + ] + }, + "blockers": [], + "adapter_ablation": true + }, + "entries": [ + {"dataset": "FB15k237LogicalQuery"}, + {"dataset": "FB15kLogicalQuery"}, + {"dataset": "NELL995LogicalQuery"}, + {"dataset": "InductiveFB15k237Query:106"}, + {"dataset": "InductiveFB15k237Query:113"}, + {"dataset": "InductiveFB15k237Query:122"}, + {"dataset": "InductiveFB15k237Query:134"}, + {"dataset": "InductiveFB15k237Query:150"}, + {"dataset": "InductiveFB15k237Query:175"}, + {"dataset": "InductiveFB15k237Query:217"}, + {"dataset": "InductiveFB15k237Query:300"}, + {"dataset": "InductiveFB15k237Query:550"}, + {"dataset": "WikiTopicsQuery:art"}, + {"dataset": "WikiTopicsQuery:award"}, + {"dataset": "WikiTopicsQuery:edu"}, + {"dataset": "WikiTopicsQuery:health"}, + {"dataset": "WikiTopicsQuery:infra"}, + {"dataset": "WikiTopicsQuery:loc"}, + {"dataset": "WikiTopicsQuery:org"}, + {"dataset": "WikiTopicsQuery:people"}, + {"dataset": "WikiTopicsQuery:sci"}, + {"dataset": "WikiTopicsQuery:sport"}, + {"dataset": "WikiTopicsQuery:tax"} + ] + }, + { + "defaults": { + "id": "trix-product-intersections-{dataset}", + "method": "trix-adapter", + "checkpoint": "checkpoints/trix/entity_prediction.pth", + "adapters": {"product": "benchmarks/adapters/trix_product_intersections.json"}, + "options": {"beam_size": 64, "row_batch_size": 1, "backend_batch_size": 1, "cache_bytes": 536870912, "raw_cache_bytes": 2147483648}, + "operators": { + "1p": "product", + "2p": "product", + "3p": "product", + "2i": "product", + "3i": "product", + "ip": "product", + "pi": "product", + "2u": "product", + "up": "product", + "2in": "product", + "3in": "product", + "inp": "product", + "pin": "product", + "pni": "product" + }, + "selection_protocol": "target-validation", + "finalized": true, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni"], + "query_batch_size": 16, + "query_order": "relation", + "reference": { + "status": "frozen transfer recipe; independent KGFM verification required", + "notes": [ + "Frozen source-trained product adapter; CQD beam 64; paired learned/identity calibration with shared atomic batches.", + "Training graphs: FB15k237, WN18RR and CoDExMedium. No target test labels select these recipes.", + "Backbone parity is checked separately; per-run KGFM evidence verifies adapter/composition against independently captured atomic logits.", + "Adapter training uses only 2i and 3i; checkpoint selection uses reserved source validation.", + "One shared recipe across both backbones and both suites, selected on validation queries of both suites; this is not target-untuned zero-shot selection." + ] + }, + "blockers": [], + "adapter_ablation": true + }, + "entries": [ + {"dataset": "FB15k237LogicalQuery"}, + {"dataset": "FB15kLogicalQuery"}, + {"dataset": "NELL995LogicalQuery"}, + {"dataset": "InductiveFB15k237Query:106"}, + {"dataset": "InductiveFB15k237Query:113"}, + {"dataset": "InductiveFB15k237Query:122"}, + {"dataset": "InductiveFB15k237Query:134"}, + {"dataset": "InductiveFB15k237Query:150"}, + {"dataset": "InductiveFB15k237Query:175"}, + {"dataset": "InductiveFB15k237Query:217"}, + {"dataset": "InductiveFB15k237Query:300"}, + {"dataset": "InductiveFB15k237Query:550"}, + {"dataset": "WikiTopicsQuery:art"}, + {"dataset": "WikiTopicsQuery:award"}, + {"dataset": "WikiTopicsQuery:edu"}, + {"dataset": "WikiTopicsQuery:health"}, + {"dataset": "WikiTopicsQuery:infra"}, + {"dataset": "WikiTopicsQuery:loc"}, + {"dataset": "WikiTopicsQuery:org"}, + {"dataset": "WikiTopicsQuery:people"}, + {"dataset": "WikiTopicsQuery:sci"}, + {"dataset": "WikiTopicsQuery:sport"}, + {"dataset": "WikiTopicsQuery:tax"} + ] + } + ] +} diff --git a/benchmarks/ultraquery/kgfm_kgicl.json b/benchmarks/ultraquery/kgfm_kgicl.json new file mode 100644 index 000000000..641f09b88 --- /dev/null +++ b/benchmarks/ultraquery/kgfm_kgicl.json @@ -0,0 +1,154 @@ +{ + "version": 1, + "answer_filter": "released", + "data_root": "KGs/UltraQuery", + "seed": 0, + "threads": 4, + "archives": { + "datasets": { + "source": "https://github.com/DeepGraphLearning/ULTRA/tree/427966ad8ed60420eef034063d44f3153addff90#datasets-1", + "checksum_status": "Every extracted runtime input is independently pinned by freeze." + }, + "models": { + "source": "https://github.com/DeepGraphLearning/ULTRA/tree/427966ad8ed60420eef034063d44f3153addff90/ckpts", + "checksum_status": "Released checkpoint and adapter bytes are independently pinned by freeze." + } + }, + "recipes": [ + { + "defaults": { + "id": "kgicl-product-intersections-{dataset}", + "method": "kgicl-adapter", + "checkpoint": "checkpoints/kgicl/KG-ICL-6L/model_best.tar", + "adapters": { + "product": "benchmarks/adapters/kgicl_product_intersections.json" + }, + "options": { + "beam_size": 64, + "row_batch_size": 8, + "backend_batch_size": 8, + "cache_bytes": 536870912, + "raw_cache_bytes": 2147483648, + "projection_cache_mb": 512 + }, + "operators": { + "1p": "product", + "2p": "product", + "3p": "product", + "2i": "product", + "3i": "product", + "ip": "product", + "pi": "product", + "2u": "product", + "up": "product", + "2in": "product", + "3in": "product", + "inp": "product", + "pin": "product", + "pni": "product" + }, + "selection_protocol": "target-validation", + "finalized": true, + "query_types": [ + "1p", + "2p", + "3p", + "2i", + "3i", + "ip", + "pi", + "2u", + "up", + "2in", + "3in", + "inp", + "pin", + "pni" + ], + "query_batch_size": 16, + "query_order": "relation", + "reference": { + "status": "KG-ICL adapter recipe, fitted and pinned; additional backbone", + "notes": [ + "Adapter fitted with the shared ULTRA/TRIX recipe (linear score-aware adapter, observed facts, bias bound 8, scale bound 2, 2i/3i source queries of FB15k237, WN18RR and CoDExMedium, source-validation checkpoint selection; training seed 2026090851) and pinned before any test run.", + "Backbone: DICE KG-ICL with the official KG-ICL-6L weights (nju-websoft/KG-ICL 6a3166e3), verified against the pinned code plus tests/fixtures/kgicl/upstream-fixes.patch; see docs/kgicl.md.", + "Prompt graphs: five examples per relation, sampled from the attached inference graph with prompt seed 0, identical for every query batch and call order; no answer-distance mask. CQD beam 64, product t-norm, paired identity control. The per-layer relation tables of every query relation are cached (projection_cache_mb 512, about 260 MiB for FB15k237); with the default 64 MiB they are recomputed on graphs with many relations, about 2.5 times slower." + ] + }, + "blockers": [], + "adapter_ablation": true + }, + "entries": [ + { + "dataset": "FB15k237LogicalQuery" + }, + { + "dataset": "FB15kLogicalQuery" + }, + { + "dataset": "NELL995LogicalQuery" + }, + { + "dataset": "InductiveFB15k237Query:106" + }, + { + "dataset": "InductiveFB15k237Query:113" + }, + { + "dataset": "InductiveFB15k237Query:122" + }, + { + "dataset": "InductiveFB15k237Query:134" + }, + { + "dataset": "InductiveFB15k237Query:150" + }, + { + "dataset": "InductiveFB15k237Query:175" + }, + { + "dataset": "InductiveFB15k237Query:217" + }, + { + "dataset": "InductiveFB15k237Query:300" + }, + { + "dataset": "InductiveFB15k237Query:550" + }, + { + "dataset": "WikiTopicsQuery:art" + }, + { + "dataset": "WikiTopicsQuery:award" + }, + { + "dataset": "WikiTopicsQuery:edu" + }, + { + "dataset": "WikiTopicsQuery:health" + }, + { + "dataset": "WikiTopicsQuery:infra" + }, + { + "dataset": "WikiTopicsQuery:loc" + }, + { + "dataset": "WikiTopicsQuery:org" + }, + { + "dataset": "WikiTopicsQuery:people" + }, + { + "dataset": "WikiTopicsQuery:sci" + }, + { + "dataset": "WikiTopicsQuery:sport" + }, + { + "dataset": "WikiTopicsQuery:tax" + } + ] + } + ] +} diff --git a/benchmarks/ultraquery/trained_baselines.json b/benchmarks/ultraquery/trained_baselines.json new file mode 100644 index 000000000..0efdc02d6 --- /dev/null +++ b/benchmarks/ultraquery/trained_baselines.json @@ -0,0 +1,96 @@ +{ + "version": 1, + "answer_filter": "released", + "data_root": "KGs/UltraQuery", + "seed": 0, + "threads": 4, + "archives": { + "datasets": {"source": "https://github.com/DeepGraphLearning/ULTRA/tree/427966ad8ed60420eef034063d44f3153addff90#datasets-1", "checksum_status": "Every extracted runtime input is independently pinned by freeze."}, + "models": {"source": "https://github.com/DeepGraphLearning/ULTRA/tree/427966ad8ed60420eef034063d44f3153addff90/ckpts", "checksum_status": "Released checkpoint and adapter bytes are independently pinned by freeze."} + }, + "recipes": [ + { + "defaults": { + "id": "qto-{dataset}", + "method": "qto", + "checkpoint": "checkpoints/ultraquery_benchmark-trained/qto-FB15kLogicalQuery/checkpoint.pt", + "options": {"threshold": 0.001, "negation_scale": 6, "row_batch_size": 32, "cache_bytes": 536870912, "reference_batching": true, "matrix_bytes": 6442450944, "host_matrix_bytes": 34359738368}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni"], + "query_batch_size": 1, + "query_order": "upstream-pickle", + "reference": { + "status": "native reference port; full reproduction pending", + "settings": "https://arxiv.org/html/2410.12537v3#A6", + "notes": ["Train with the pinned QTO FB15k recipe, then verify against independent QTO inference."], + "inference_graph": "train" + }, + "blockers": [], + "training": "checkpoints/ultraquery_benchmark-trained/qto-FB15kLogicalQuery/training.json" + }, + "entries": [{"dataset": "FB15kLogicalQuery"}] + }, + { + "defaults": { + "id": "inductive-gnnqe-{dataset}", + "method": "inductive-gnnqe", + "options": {"logic": "product"}, + "query_types": ["1p", "2p", "3p", "2i", "3i", "ip", "pi", "2u", "up", "2in", "3in", "inp", "pin", "pni"], + "query_batch_size": 64, + "query_order": "upstream-pickle", + "reference": { + "status": "requires locally trained v2 checkpoint and independent validation parity", + "settings": "https://github.com/DeepGraphLearning/InductiveQE/blob/ab49a0e64e3449e687e484a42269d9898e1d1325/config/complex_query/gnnqe_main.yaml", + "notes": ["Train on corrected Zenodo 7306046 version 2.0; validation-only model selection."] + }, + "blockers": [] + }, + "entries": [ + { + "dataset": "InductiveFB15k237Query:106", + "checkpoint": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:106/checkpoint.pt", + "training": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:106/training.json" + }, + { + "dataset": "InductiveFB15k237Query:113", + "checkpoint": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:113/checkpoint.pt", + "training": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:113/training.json" + }, + { + "dataset": "InductiveFB15k237Query:122", + "checkpoint": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:122/checkpoint.pt", + "training": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:122/training.json" + }, + { + "dataset": "InductiveFB15k237Query:134", + "checkpoint": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:134/checkpoint.pt", + "training": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:134/training.json" + }, + { + "dataset": "InductiveFB15k237Query:150", + "checkpoint": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:150/checkpoint.pt", + "training": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:150/training.json" + }, + { + "dataset": "InductiveFB15k237Query:175", + "checkpoint": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:175/checkpoint.pt", + "training": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:175/training.json" + }, + { + "dataset": "InductiveFB15k237Query:217", + "checkpoint": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:217/checkpoint.pt", + "training": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:217/training.json" + }, + { + "dataset": "InductiveFB15k237Query:300", + "checkpoint": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:300/checkpoint.pt", + "training": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:300/training.json" + }, + { + "dataset": "InductiveFB15k237Query:550", + "checkpoint": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:550/checkpoint.pt", + "training": "checkpoints/ultraquery_benchmark-trained/inductive-gnnqe-InductiveFB15k237Query:550/training.json" + } + ] + } + ] +} diff --git a/dicee/config.py b/dicee/config.py index e018b7e7d..d0a26ddc3 100644 --- a/dicee/config.py +++ b/dicee/config.py @@ -55,6 +55,17 @@ def __init__(self, **kwargs): self.flock_compact_state: bool = True self.flock_compile_sampler: bool = True self.flock_pack_walks: bool = True + self.kgicl_checkpoint: Optional[str] = None + self.kgicl_dim: int = 32 + self.kgicl_attn_dim: int = 5 + self.kgicl_num_layers: int = 6 + self.kgicl_prompt_layers: int = 3 + self.kgicl_prompt_hops: int = 3 + self.kgicl_shots: int = 5 + self.kgicl_prompt_open_nodes: int = 50 + self.kgicl_prompt_seed: int = 0 + self.kgicl_masked_distances: Optional[list] = None + self.kgicl_query_batch_size: int = 8 self.graph_inference_backend: str = 'auto' self.graph_relation_cache_mb: int = 64 self.graph_projection_cache_mb: int = 64 diff --git a/dicee/evaluation/_filtering.py b/dicee/evaluation/_filtering.py index ee13c6cb0..283b21b12 100644 --- a/dicee/evaluation/_filtering.py +++ b/dicee/evaluation/_filtering.py @@ -52,7 +52,7 @@ def rank(self, predictions: torch.Tensor, target_idx: SupportsInt, if exclude_target: filters.discard(target_idx) if filters: - filtered[list(filters)] = -np.Inf + filtered[list(filters)] = -np.inf if self.tie_policy == "sort": # Preserve DICE's original per-vector torch.sort ordering. @@ -90,7 +90,7 @@ def rank_batch(self, predictions: torch.Tensor, target_indices: Iterable[Support for i, (target, filters) in enumerate(zip(target_indices, filter_indices_list)): target = int(target) if len(filters): - filtered[i, list(filters)] = -np.Inf + filtered[i, list(filters)] = -np.inf filtered[i, target] = predictions[i, target] order = torch.sort(filtered, dim=1, descending=True).indices return [int(torch.where(row == int(target))[0].item()) + 1 diff --git a/dicee/knowledge_graph_embeddings.py b/dicee/knowledge_graph_embeddings.py index abe9eb034..03d30fcf7 100644 --- a/dicee/knowledge_graph_embeddings.py +++ b/dicee/knowledge_graph_embeddings.py @@ -33,8 +33,14 @@ def __str__(self): def to(self, device: str) -> None: if "cpu" not in device and "cuda" not in device: raise ValueError(f"Device must be either cpu or cuda, got {device!r}") + self.clear_query_cache() self.model.to(device) + def clear_query_cache(self): + """Release retained CQD rows and their model references.""" + self._query_engine = None + self._query_engine_key = None + def get_transductive_entity_embeddings(self, indices: Union[torch.LongTensor, List[str]], as_pytorch=False, @@ -722,588 +728,76 @@ def triple_score(self, h: Optional[Union[List[str], str]] = None, r: Optional[Un else: return torch.sigmoid(self.model(x)) - def return_multi_hop_query_results(self, aggregated_query_for_all_entities, k: int, only_scores): - # @TODO: refactor by torchargmax(aggregated_query_for_all_entities) - if only_scores: - return aggregated_query_for_all_entities - # from idx obtain entity str - return sorted([(ei, s) for ei, s in zip(self.entity_to_idx.keys(), aggregated_query_for_all_entities)], - key=lambda x: x[1], reverse=True)[:k] - - def single_hop_query_answering(self, query: tuple, only_scores: bool = True, k: Optional[int] = None, - use_logits: bool = True): - h, r = query - result = self.predict(h=h, r=r[0], logits=use_logits).squeeze() + def return_multi_hop_query_results(self, scores, k: int, only_scores): if only_scores: - """ do nothing""" - else: - query_score_of_all_entities = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), result)] - result = sorted(query_score_of_all_entities, key=lambda x: x[1], reverse=True)[:k] - return result - - def answer_multi_hop_query( - self, - query_type: Optional[str] = None, - query: Optional[Tuple[Union[str, Tuple[str, str]], ...]] = None, - queries: Optional[List[Tuple[Union[str, Tuple[str, str]], ...]]] = None, - tnorm: str = "prod", - neg_norm: str = "standard", - lambda_: float = 0.0, - k: int = 10, - only_scores: bool = False, - use_logits: bool = False - ) -> Union[List[Tuple[str, torch.Tensor]], List[List[Tuple[str, torch.Tensor]]]]: - """ - Answer multi-hop EPFO (Existential Positive First-Order) queries. - - Supports 9 query types: 1p, 2p, 3p, 2i, 3i, ip, pi, 2u, up. - See docs/guides/multi_hop_queries.md for detailed query patterns. - - Args: - query_type: Query pattern name. One of: - - 1p: (e, (r,)) # One-hop - - 2p: (e, (r1, r2)) # Two-hop - - 3p: (e, (r1, r2, r3)) # Three-hop - - 2i: ((e1, (r1,)), (e2, (r2,))) # Two-way intersection - - 3i: ((e1, (r1,)), (e2, (r2,)), (e3, (r3,))) # Three-way intersection - - ip: (((e1, (r1,)), (e2, (r2,))), (r3,)) # Intersection + projection - - pi: ((e, (r1, r2)), (r3,)) # Projection + intersection (2i meets 2p) - - 2u: ((e1, (r1,)), (e2, (r2,))) # Two-way union - - up: ((e, (r1, r2)), (e, (r3,))) # Union + projection - query: Single query tuple matching the query_type pattern. - queries: Batch of queries. If provided, query must be None. - tnorm: T-norm for intersection/union. Options: "prod", "min". - neg_norm: Negation norm. Options: "standard", "sugeno", "yager". - lambda_: Parameter for sugeno and yager negation (0.0-1.0). - k: Number of top answer entities to return. - only_scores: If True, return only scores tensor. If False, return (entity, score) tuples. - use_logits: If False (default), combine sigmoid probabilities in [0, 1], which is what the - T-norms and negation norms are defined on. If True, combine raw logits; the product - T-norm and the negation norms then give meaningless rankings for negative scores. + return scores + order = torch.argsort(scores, descending=True, stable=True)[:k] + names = {index: name for name, index in self.entity_to_idx.items()} + return [(names[int(index)], scores[index]) for index in order] - Returns: - For single query: List[(entity, score), ...] of top-k answers. - For batch queries: List of such lists, one per query. + def single_hop_query_answering(self, query: tuple, only_scores: bool = True, + k: Optional[int] = None, use_logits: bool = False): + return self.answer_multi_hop_query('1p', query, k=len(self.entity_to_idx) if k is None else k, + only_scores=only_scores, use_logits=use_logits) - Raises: - ValueError: If query_type is not in {1p, 2p, 3p, 2i, 3i, ip, pi, 2u, up}. - AssertionError: If query structure doesn't match query_type pattern. - - Examples: - >>> # 1p: Find entities located in Asia - >>> model.answer_multi_hop_query( - ... query_type="1p", - ... query=("Asia", ("isLocatedIn",)), - ... k=5 - ... ) - [("Mongolia", 0.92), ("China", 0.89), ...] - - >>> # 2p: Two-hop query (e.g., "capital of countries in Europe") - >>> model.answer_multi_hop_query( - ... query_type="2p", - ... query=("Europe", ("isLocatedIn", "hasCapital")), - ... k=3 - ... ) - [("Paris", 0.85), ("Berlin", 0.82), ...] - - >>> # 2i: Intersection query - >>> model.answer_multi_hop_query( - ... query_type="2i", - ... query=(("Asia", ("isLocatedIn",)), ("Mountains", ("hasGeography",))), - ... k=5 - ... ) - [("Nepal", 0.78), ("Tibet", 0.65), ...] - - See Also: - - docs/guides/multi_hop_queries.md: Complete guide with all query patterns - - tests/test_answer_multi_hop_query.py: Usage examples + def answer_multi_hop_query( + self, query_type=None, query=None, queries=None, tnorm="prod", + neg_norm="standard", lambda_=0.0, k=10, only_scores=False, + use_logits=False, *, beam_size=None, context=None, adapter=None, + observed_mix=None, row_batch_size=8, cache_bytes=64 * 1024 * 1024, + seed=0, samples=None, executor='cqd', + ): + """Answer the 14 standard query shapes and the +H four-step 4p/4i shapes. + + All entity-prediction models use one evaluator, including ULTRA, TRIX, + and Flock. Scores default to sigmoid memberships; ``use_logits=True`` + explicitly requests legacy raw-score algebra (without an adapter). The + T-norms and negation norms are defined on memberships in [0, 1], so with + raw logits the product T-norm and the negation norms give meaningless + rankings for negative scores, and such calls log a warning. + ``k`` limits returned answers; ``beam_size`` controls existential search + independently and defaults to max(1, k). ``only_scores`` returns all + entity scores in numeric ID order. Ties use entity ID order. + ``executor='qto'`` uses exact projections without a beam limit. + + Pass QueryContext for transductive observed-edge/feature support; graph + models use their attached context. Pass QueryScoreAdapter for a learned + transform. Context must never contain held-out evaluation answers. """ + from .query_answering import QueryAnswerer + from .query_answering._query import index_query + if (query is None) == (queries is None): + raise ValueError("Provide exactly one of 'query' or 'queries'") + if type(k) is not int or k < 0: + raise ValueError('k must be a nonnegative integer') if use_logits and (tnorm == "prod" or (query_type is not None and "n" in query_type)): logger.warning("use_logits=True with tnorm='prod' or a negation query combines unbounded scores; " "use_logits=False (sigmoid probabilities) or tnorm='min' for meaningful rankings.") - - if queries is not None: - if query is not None: - raise ValueError("Provide either 'query' or 'queries', not both") - results = [] - for i in queries: - results.append( - self.answer_multi_hop_query(query_type=query_type, query=i, tnorm=tnorm, neg_norm=neg_norm, - lambda_=lambda_, k=k, only_scores=only_scores, - use_logits=use_logits)) - return results - - if not (len(self.entity_to_idx) >= k >= 0): - raise ValueError(f"k must satisfy 0 <= k <= {len(self.entity_to_idx)}, got {k}") - - query_name_dict = { - ("e", ("r",)): "1p", - ("e", ("r", "r")): "2p", - ("e", ("r", "r", "r",),): "3p", - (("e", ("r",)), ("e", ("r",))): "2i", - (("e", ("r",)), ("e", ("r",)), ("e", ("r",))): "3i", - ((("e", ("r",)), ("e", ("r",))), ("r",)): "ip", - (("e", ("r", "r")), ("e", ("r",))): "pi", - # negation - (("e", ("r",)), ("e", ("r", "n"))): "2in", - (("e", ("r",)), ("e", ("r",)), ("e", ("r", "n"))): "3in", - ((("e", ("r",)), ("e", ("r", "n"))), ("r",)): "inp", - (("e", ("r", "r")), ("e", ("r", "n"))): "pin", - (("e", ("r", "r", "n")), ("e", ("r",))): "pni", - - # union - (("e", ("r",)), ("e", ("r",)), ("u",)): "2u", - ((("e", ("r",)), ("e", ("r",)), ("u",)), ("r",)): "up", - - } - - # Create an inverse mapping - inverse_query_name_dict = {v: k for k, v in query_name_dict.items()} - - # Look up the corresponding query_structure - if query_type in inverse_query_name_dict: - query_structure = inverse_query_name_dict[query_type] - else: - supported_queries = sorted([k for k in inverse_query_name_dict.keys() if 'n' not in k]) - raise ValueError( - f"Invalid query type: '{query_type}'\\n" - f"\\nSupported query types:\\n" - f" - Basic: {', '.join(supported_queries[:7])}\\n" - f" - Negation: {', '.join([k for k in sorted(inverse_query_name_dict.keys()) if 'n' in k])}\\n" - f"\\nExamples:\\n" - f" - 1p: (e, (r,))\\n" - f" - 2p: (e, (r1, r2))\\n" - f" - 2i: ((e1, (r1,)), (e2, (r2,)))\\n" - f" - 2u: ((e1, (r1,)), (e2, (r2,)), ('u',))\\n" - f"\\nSee docs/guides/multi_hop_queries.md for complete guide\\n" - f"See tests/test_answer_multi_hop_query.py for usage examples\\n" - ) - - # 1p - if query_structure == ("e", ("r",)): - return self.single_hop_query_answering(query, only_scores, k, use_logits=use_logits) - # 2p - elif query_structure == ("e", ("r", "r",)): - # ?M : \exist A. r1(e,A) \land r2(A,M) - e, (r1, r2) = query - top_k_scores1 = [] - atom2_scores = [] - # (1) Iterate over top k substitutes of A in the first hop query: r1(e,A) s.t. A<-a - for top_k_entity, score_of_e_r1_a in self.answer_multi_hop_query(query_type="1p", query=(e, (r1,)), - only_scores=False, tnorm=tnorm, k=k, - use_logits=use_logits): - # (1.1) Store scores of (e, r1, a) s.t. a is a substitute of A and a is a top ranked entity. - top_k_scores1.append(score_of_e_r1_a) - # (1.2) Compute scores for (a, r2, M): Replace predict with answer_multi_hop_query. - atom2_scores.append(self.predict(h=top_k_entity, r=r2, logits=use_logits)) - # (2) k by E tensor - atom2_scores = torch.vstack(atom2_scores) - kk, E = atom2_scores.shape - # Sanity checking - assert k == kk - # Top k scores for all replacement of A. torch.Size([k,1]) - top_k_scores1 = torch.FloatTensor(top_k_scores1).reshape(k, 1) - # k x E - top_k_scores1 = top_k_scores1.repeat(1, E) - # E scores - aggregated_query_for_all_entities, _ = torch.max(self.t_norm(top_k_scores1, atom2_scores, tnorm), dim=0) - return self.return_multi_hop_query_results(aggregated_query_for_all_entities, k, only_scores) - # 3p - elif query_structure == ("e", ("r", "r", "r",)): - head1, (relation1, relation2, relation3) = query - top_k_scores1 = [] - atom_scores = [] - # (1) Iterate over top k substitutes of A in the first hop query: r1(e,A) s.t. A<-a - for top_k_entity, score_of_e_r1_a in self.answer_multi_hop_query(query_type="2p", - query=(head1, (relation1, relation2)), - tnorm=tnorm, - k=k, - use_logits=use_logits): - top_k_scores1.append(score_of_e_r1_a) - # () Scores for all entities E - atom_scores.append(self.predict(h=[top_k_entity], r=[relation3], logits=use_logits)) - - # (2) k by E tensor - atom_scores = torch.vstack(atom_scores) - kk, E = atom_scores.shape - # Sanity checking - assert k == kk - # Top k scores for all replacement of A. torch.Size([k,1]) - top_k_scores1 = torch.FloatTensor(top_k_scores1).reshape(k, 1) - # k x E - top_k_scores1 = top_k_scores1.repeat(1, E) - # E scores - aggregated_query_for_all_entities, _ = torch.max(self.t_norm(top_k_scores1, atom_scores, tnorm), dim=0) - return self.return_multi_hop_query_results(aggregated_query_for_all_entities, k, only_scores) - - # 2in - elif query_structure == (("e", ("r",)), ("e", ("r", "n"))): - # entity_scores = scores_2in(query, tnorm, neg_norm, lambda_) - head1, relation1 = query[0] - head2, relation2 = query[1] - - # Calculate entity scores for each query - # Get scores for the first atom (positive) - atom1_scores = self.predict(h=[head1], r=[relation1[0]], logits=use_logits).squeeze() - # Get scores for the second atom (negative) - # if neg_norm == "standard": - predictions = self.predict(h=[head2], r=[relation2[0]], logits=use_logits).squeeze() - atom2_scores = self.negnorm(predictions, lambda_, neg_norm) - - assert len(atom1_scores) == len(self.entity_to_idx) - - combined_scores = self.t_norm(atom1_scores, atom2_scores, tnorm) - if only_scores: - return combined_scores - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), combined_scores)] - return sorted(entity_scores, key=lambda x: x[1], reverse=True) - # 3in - elif query_structure == (("e", ("r",)), ("e", ("r",)), ("e", ("r", "n"))): - # entity_scores = scores_3in(model, query, tnorm, neg_norm, lambda_) - head1, relation1 = query[0] - head2, relation2 = query[1] - head3, relation3 = query[2] - - # Calculate entity scores for each query - # Get scores for the first atom (positive) - atom1_scores = self.predict(h=[head1], r=[relation1[0]], logits=use_logits).squeeze() - # Get scores for the second atom (negative) - # modelling standard negation (1-x) - atom2_scores = self.predict(h=[head2], r=[relation2[0]], logits=use_logits).squeeze() - # Get scores for the third atom - # if neg_norm == "standard": - predictions = self.predict(h=[head3], r=[relation3[0]], logits=use_logits).squeeze() - atom3_scores = self.negnorm(predictions, lambda_, neg_norm) - - assert len(atom1_scores) == len(self.entity_to_idx) - - inter_scores = self.t_norm(atom1_scores, atom2_scores, tnorm) - combined_scores = self.t_norm(inter_scores, atom3_scores, tnorm) - if only_scores: - return combined_scores - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), combined_scores)] - return sorted(entity_scores, key=lambda x: x[1], reverse=True) - # pni - elif query_structure == (("e", ("r", "r", "n")), ("e", ("r",))): - # entity_scores = scores_pni(model, query, tnorm, neg_norm, lambda_, k_) - head1, (relation1, relation2, _) = query[0] - head3, relation3 = query[1] - # Calculate entity scores for each query - # Get scores for the first atom - atom1_scores = self.predict(h=[head1], r=[relation1], logits=use_logits).squeeze() - - assert len(atom1_scores) == len(self.entity_to_idx) - # sort atom1_scores in descending order and get the top k entities indices - top_k_scores1, top_k_indices = torch.topk(atom1_scores, k) - - # using model.entity_to_idx.keys() take the name of entities from topk heads 2 - entity_to_idx_keys = list(self.entity_to_idx.keys()) - top_k_heads = [entity_to_idx_keys[idx.item()] for idx in top_k_indices] - - # Get scores for the second atom - # Initialize an empty tensor - atom2_scores = torch.empty(0, len(self.entity_to_idx)).to(atom1_scores.device) - - # Get scores for the second atom - for head2 in top_k_heads: - # The score tensor for the current head2 - atom2_score = self.predict(h=[head2], r=[relation2], logits=use_logits) - neg_atom2_score = self.negnorm(atom2_score, lambda_, neg_norm) - # Concatenate the score tensor for the current head2 with the previous scores - atom2_scores = torch.cat([atom2_scores, neg_atom2_score], dim=0) - - topk_scores1_expanded = top_k_scores1.view(-1, 1).repeat(1, atom2_scores.shape[1]) - - inter_scores = self.t_norm(topk_scores1_expanded, atom2_scores, tnorm) - - scores_2pn_query, _ = torch.max(inter_scores, dim=0) - scores_1p_query = self.predict(h=[head3], r=[relation3[0]], logits=use_logits).squeeze() - - combined_scores = self.t_norm(scores_2pn_query, scores_1p_query, tnorm) - if only_scores: - return combined_scores - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), combined_scores)] - return sorted(entity_scores, key=lambda x: x[1], reverse=True) - # pin - elif query_structure == (("e", ("r", "r")), ("e", ("r", "n"))): - # entity_scores = scores_pin(model, query, tnorm, neg_norm, lambda_, k_) - head1, (relation1, relation2) = query[0] - head3, relation3 = query[1] - # Calculate entity scores for each query - # Get scores for the first atom - atom1_scores = self.predict(h=[head1], r=[relation1], logits=use_logits).squeeze() - - assert len(atom1_scores) == len(self.entity_to_idx) - - # sort atom1_scores in descending order and get the top k entities indices - top_k_scores1, top_k_indices = torch.topk(atom1_scores, k) - - # using model.entity_to_idx.keys() take the name of entities from topk heads 2 - entity_to_idx_keys = list(self.entity_to_idx.keys()) - top_k_heads = [entity_to_idx_keys[idx.item()] for idx in top_k_indices] - - # Initialize an empty tensor - atom2_scores = torch.empty(0, len(self.entity_to_idx)).to(atom1_scores.device) - - # Get scores for the second atom - for head2 in top_k_heads: - # The score tensor for the current head2 - atom2_score = self.predict(h=[head2], r=[relation2], logits=use_logits) - # Concatenate the score tensor for the current head2 with the previous scores - atom2_scores = torch.cat([atom2_scores, atom2_score], dim=0) - - topk_scores1_expanded = top_k_scores1.view(-1, 1).repeat(1, atom2_scores.shape[1]) - - inter_scores = self.t_norm(topk_scores1_expanded, atom2_scores, tnorm) - - scores_2p_query, _ = torch.max(inter_scores, dim=0) - - scores_1p_query = self.predict(h=[head3], r=[relation3[0]], logits=use_logits).squeeze() - # taking negation for the e,(r,n) part of query - neg_scores_1p_query = self.negnorm(scores_1p_query, lambda_, neg_norm) - combined_scores = self.t_norm(scores_2p_query, neg_scores_1p_query, tnorm) - if only_scores: - return combined_scores - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), combined_scores)] - return sorted(entity_scores, key=lambda x: x[1], reverse=True) - # inp - elif query_structure == ((("e", ("r",)), ("e", ("r", "n"))), ("r",)): - # entity_scores = scores_inp(model, query, tnorm, neg_norm, lambda_, k_) - head1, relation1 = query[0][0] - head2, relation2 = query[0][1] - relation_1p = query[1] - - # Calculate entity scores for each query - # Get scores for the first atom (positive) - atom1_scores = self.predict(h=[head1], r=[relation1[0]], logits=use_logits).squeeze() - # Get scores for the second atom (negative) - # if neg_norm == "standard": - predictions = self.predict(h=[head2], r=[relation2[0]], logits=use_logits).squeeze() - atom2_scores = self.negnorm(predictions, lambda_, neg_norm) - - assert len(atom1_scores) == len(self.entity_to_idx) - - scores_2in_query = self.t_norm(atom1_scores, atom2_scores, tnorm) - - # sort atom1_scores in descending order and get the top k entities indices - top_k_scores1, top_k_indices = torch.topk(scores_2in_query, k) - - # using model.entity_to_idx.keys() take the name of entities from topk heads 2 - entity_to_idx_keys = list(self.entity_to_idx.keys()) - top_k_heads = [entity_to_idx_keys[idx.item()] for idx in top_k_indices] - - # Get scores for the second atom - # Initialize an empty tensor - atom3_scores = torch.empty(0, len(self.entity_to_idx)).to(scores_2in_query.device) - - # Get scores for the second atom - for head3 in top_k_heads: - # The score tensor for the current head2 - atom3_score = self.predict(h=[head3], r=[relation_1p[0]], logits=use_logits) - # Concatenate the score tensor for the current head2 with the previous scores - atom3_scores = torch.cat([atom3_scores, atom3_score], dim=0) - - topk_scores1_expanded = top_k_scores1.view(-1, 1).repeat(1, atom3_scores.shape[1]) - - combined_scores = self.t_norm(topk_scores1_expanded, atom3_scores, tnorm) - - res, _ = torch.max(combined_scores, dim=0) - if only_scores: - return res - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), res)] - return sorted(entity_scores, key=lambda x: x[1], reverse=True) - # 2i - elif query_structure == (("e", ("r",)), ("e", ("r",))): - # entity_scores = scores_2i(model, query, tnorm) - head1, relation1 = query[0] - head2, relation2 = query[1] - - # Calculate entity scores for each query - # Get scores for the first atom - atom1_scores = self.predict(h=[head1], r=[relation1[0]], logits=use_logits).squeeze() - # Get scores for the second atom - atom2_scores = self.predict(h=[head2], r=[relation2[0]], logits=use_logits).squeeze() - - assert len(atom1_scores) == len(self.entity_to_idx) - - combined_scores = self.t_norm(atom1_scores, atom2_scores, tnorm) + key = (id(self.model), id(context), id(adapter), observed_mix, row_batch_size, seed, samples) + if key != getattr(self, '_query_engine_key', None): + self._query_engine = QueryAnswerer(self.model, context=context, adapter=adapter, observed_mix=observed_mix, + row_batch_size=row_batch_size, cache_bytes=cache_bytes, seed=seed, samples=samples) + self._query_engine_key = key + engine = self._query_engine + if cache_bytes < 0: + raise ValueError('Cache size must be nonnegative') + engine.cache_bytes = cache_bytes + beam = max(1, k) if beam_size is None else beam_size + names = {index: name for name, index in self.entity_to_idx.items()} if not only_scores else None + + def answer(item): + indexed = index_query(query_type, item, self.entity_to_idx, self.relation_to_idx) + scores = engine.predict(indexed, beam_size=beam, tnorm=tnorm, neg_norm=neg_norm, + lambda_=lambda_, use_logits=use_logits, return_log_scores=not use_logits, executor=executor) if only_scores: - return combined_scores - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), combined_scores)] - return sorted(entity_scores, key=lambda x: x[1], reverse=True) - # 3i - elif query_structure == (("e", ("r",)), ("e", ("r",)), ("e", ("r",))): - # entity_scores = scores_3i(model, query, tnorm) - head1, relation1 = query[0] - head2, relation2 = query[1] - head3, relation3 = query[2] - # Calculate entity scores for each query - # Get scores for the first atom - atom1_scores = self.predict(h=[head1], r=[relation1[0]], logits=use_logits).squeeze() - # Get scores for the second atom - atom2_scores = self.predict(h=[head2], r=[relation2[0]], logits=use_logits).squeeze() - # Get scores for the third atom - atom3_scores = self.predict(h=[head3], r=[relation3[0]], logits=use_logits).squeeze() - - assert len(atom1_scores) == len(self.entity_to_idx) - - inter_scores = self.t_norm(atom1_scores, atom2_scores, tnorm) - combined_scores = self.t_norm(inter_scores, atom3_scores, tnorm) - if only_scores: - return combined_scores - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), combined_scores)] - return sorted(entity_scores, key=lambda x: x[1], reverse=True) - # pi - elif query_structure == (("e", ("r", "r")), ("e", ("r",))): - # entity_scores = scores_pi(model, query, tnorm, k_) - head1, (relation1, relation2) = query[0] - head3, relation3 = query[1] - # Calculate entity scores for each query - # Get scores for the first atom - atom1_scores = self.predict(h=[head1], r=[relation1], logits=use_logits).squeeze() - - assert len(atom1_scores) == len(self.entity_to_idx) - # sort atom1_scores in descending order and get the top k entities indices - top_k_scores1, top_k_indices = torch.topk(atom1_scores, k) - - # using model.entity_to_idx.keys() take the name of entities from topk heads 2 - entity_to_idx_keys = list(self.entity_to_idx.keys()) - top_k_heads = [entity_to_idx_keys[idx.item()] for idx in top_k_indices] - - # Initialize an empty tensor - atom2_scores = torch.empty(0, len(self.entity_to_idx)).to(atom1_scores.device) - - # Get scores for the second atom - for head2 in top_k_heads: - # The score tensor for the current head2 - atom2_score = self.predict(h=[head2], r=[relation2], logits=use_logits).unsqueeze(0) - # Concatenate the score tensor for the current head2 with the previous scores - atom2_scores = torch.cat([atom2_scores, atom2_score], dim=0) - - topk_scores1_expanded = top_k_scores1.view(-1, 1).repeat(1, atom2_scores.shape[1]) - - inter_scores = self.t_norm(topk_scores1_expanded, atom2_scores, tnorm) - - scores_2p_query, _ = torch.max(inter_scores, dim=0) - - scores_1p_query = self.predict(h=[head3], r=[relation3[0]], logits=use_logits).squeeze() - - combined_scores = self.t_norm(scores_2p_query, scores_1p_query, tnorm) - if only_scores: - return combined_scores - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), combined_scores)] - return sorted(entity_scores, key=lambda x: x[1], reverse=True) - # ip - elif query_structure == ((("e", ("r",)), ("e", ("r",))), ("r",)): - # entity_scores = scores_ip(model, query, tnorm, k_) - head1, relation1 = query[0][0] - head2, relation2 = query[0][1] - relation_1p = query[1] - # Calculate entity scores for each query - # Get scores for the first atom - atom1_scores = self.predict(h=[head1], r=[relation1[0]], logits=use_logits).squeeze() - # Get scores for the second atom - atom2_scores = self.predict(h=[head2], r=[relation2[0]], logits=use_logits).squeeze() - - assert len(atom1_scores) == len(self.entity_to_idx) - - scores_2i_query = self.t_norm(atom1_scores, atom2_scores, tnorm) - # Get the top k entities from the 2i query - - # sort atom1_scores in descending order and get the top k entities indices - top_k_scores1, top_k_indices = torch.topk(scores_2i_query, k) - - # using model.entity_to_idx.keys() take the name of entities from topk heads - entity_to_idx_keys = list(self.entity_to_idx.keys()) - top_k_heads = [entity_to_idx_keys[idx.item()] for idx in top_k_indices] - - # Get scores for the second atom - # Initialize an empty tensor - atom3_scores = torch.empty(0, len(self.entity_to_idx)).to(scores_2i_query.device) - - # Get scores for the second atom - for head3 in top_k_heads: - # The score tensor for the current head2 - atom3_score = self.predict(h=[head3], r=[relation_1p[0]], logits=use_logits).unsqueeze(0) - - # Concatenate the score tensor for the current head2 with the previous scores - atom3_scores = torch.cat([atom3_scores, atom3_score], dim=0) - - topk_scores1_expanded = top_k_scores1.view(-1, 1).repeat(1, atom3_scores.shape[1]) - - combined_scores = self.t_norm(topk_scores1_expanded, atom3_scores, tnorm) - res, _ = torch.max(combined_scores, dim=0) - if only_scores: - return res - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), res)] - return sorted(entity_scores, key=lambda x: x[1], reverse=True) - # disjunction - # 2u - elif query_structure == (("e", ("r",)), ("e", ("r",)), ("u",)): - # entity_scores = scores_2u(model, query, tnorm) - head1, relation1 = query[0] - head2, relation2 = query[1] - - # Calculate entity scores for each query - # Get scores for the first atom - atom1_scores = self.predict(h=[head1], r=[relation1[0]], logits=use_logits).squeeze() - # Get scores for the second atom - atom2_scores = self.predict(h=[head2], r=[relation2[0]], logits=use_logits).squeeze() - - assert len(atom1_scores) == len(self.entity_to_idx) - - combined_scores = self.t_conorm(atom1_scores, atom2_scores, tnorm) - if only_scores: - return combined_scores - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), combined_scores)] - entity_scores = sorted(entity_scores, key=lambda x: x[1], reverse=True) + return scores if use_logits else scores.exp() + # Rank in log space before converting to memberships, avoiding underflow ties. + order = torch.argsort(scores, descending=True, stable=True)[:k] + values = scores if use_logits else scores.exp() + return [(names[int(index)], values[index]) for index in order] - return entity_scores - # up - # here the second tnorm is for t-conorm (used in pairs) - elif query_structure == ((("e", ("r",)), ("e", ("r",)), ("u",)), ("r",)): - # entity_scores = scores_up(model, query, tnorm, tnorm, k_) - head1, relation1 = query[0][0] - head2, relation2 = query[0][1] - relation_1p = query[1] - - # Get scores for the first atom - atom1_scores = self.predict(h=[head1], r=[relation1[0]], logits=use_logits).squeeze() - - # Get scores for the second atom - atom2_scores = self.predict(h=[head2], r=[relation2[0]], logits=use_logits).squeeze() - - assert len(atom1_scores) == len(self.entity_to_idx) - - scores_2u_query = self.t_conorm(atom1_scores, atom2_scores, tnorm) - - # Sort atom1_scores in descending order and get the top k entities indices - top_k_scores1, top_k_indices = torch.topk(scores_2u_query, k) - - # Using model.entity_to_idx.keys() take the name of entities from topk heads - entity_to_idx_keys = list(self.entity_to_idx.keys()) - top_k_heads = [entity_to_idx_keys[idx.item()] for idx in top_k_indices] - - # Initialize an empty tensor - atom3_scores = torch.empty(0, len(self.entity_to_idx)).to(scores_2u_query.device) - - for head3 in top_k_heads: - # The score tensor for the current head3 - atom3_score = self.predict(h=[head3], r=[relation_1p[0]], logits=use_logits).unsqueeze(0) - - # Concatenate the score tensor for the current head3 with the previous scores - atom3_scores = torch.cat([atom3_scores, atom3_score], dim=0) - - topk_scores1_expanded = top_k_scores1.view(-1, 1).repeat(1, atom3_scores.shape[1]) - combined_scores = self.t_norm(topk_scores1_expanded, atom3_scores, tnorm) - res, _ = torch.max(combined_scores, dim=0) - if only_scores: - return res - entity_scores = [(ei, s) for ei, s in zip(self.entity_to_idx.keys(), res)] - return sorted(entity_scores, key=lambda x: x[1], reverse=True) - else: - raise RuntimeError(f"Incorrect query_structure {query_structure}") + return [answer(item) for item in queries] if queries is not None else answer(query) def find_missing_triples(self, confidence: float, entities: Optional[List[str]] = None, relations: Optional[List[str]] = None, topk: int = 10, diff --git a/dicee/models/__init__.py b/dicee/models/__init__.py index 45f71cd4c..95bcdb9e0 100644 --- a/dicee/models/__init__.py +++ b/dicee/models/__init__.py @@ -12,3 +12,4 @@ from .trix import TRIX, TRIXRelation # noqa from .flock import Flock, FlockRelation # noqa +from .kgicl import KGICL # noqa diff --git a/dicee/models/_fused_kgicl.py b/dicee/models/_fused_kgicl.py new file mode 100644 index 000000000..065108a70 --- /dev/null +++ b/dicee/models/_fused_kgicl.py @@ -0,0 +1,110 @@ +"""Optional KG-ICL inference kernels; training and unsupported settings use PyTorch. + +The fused path keeps a dense node-major ``[entities, batch, dim]`` state with an +active mask. Messages leave only active entities, so its scores equal +upstream's hop-by-hop expansion. Triton is imported only on the CUDA path. +""" +import torch +from torch.nn import functional as F + +from ._fused_message import CSRLayout +from ._inference import to_device + +ATTENTION = 16 # Padded attention width of the fused kernels. + + +def fused_reasoning_supported(model, edges): + """Whether batched Triton reasoning applies; ``backend='triton'`` raises when it cannot.""" + backend = getattr(model, '_inference_backend', 'auto') + weight = model.W_final.weight + supported = (edges[0] is model.edge_index and edges[1] is model.edge_type and weight.is_cuda + and weight.dtype == torch.float32 and model.dim <= 128 and model.attn_dim <= ATTENTION + and model.num_entities < 2**31 and model.edge_type.numel() < 2**31) + if backend == 'torch' or not supported: + if backend == 'triton' and not supported: + raise RuntimeError('The triton KG-ICL backend requires float32 CUDA inference on the attached graph') + return False + try: + from . import _triton_kgicl # noqa: F401 + except ImportError: + if backend == 'triton': + raise RuntimeError('The triton inference backend requires Triton') from None + return False + return True + + +def layer_tables(model, relation, prompt): + """Per-layer relation states, their attention projections and the query term of one relation. + + Computed for a single relation at a time, so values never depend on which + other relations share a query batch. + """ + rel, result = prompt, [] + for i, layer in enumerate(model.gnn_layers): + rel = model.layer_norms_rel[i](rel + F.relu(model.rel_transfer[i](rel))) + result.append((rel, F.pad(layer.Wr_attn(rel), (0, ATTENTION - model.attn_dim)), + F.pad(layer.Wqr_attn(rel[relation]), (0, ATTENTION - model.attn_dim)))) + rel = model.layer_norms_rel[i](rel) + return result + + +def tile_layout(targets, sources, types, num_nodes, edges, hub_tiles): + """Edges grouped by target in ``edges``-wide tiles; rows with more than ``hub_tiles`` tiles are hubs. + + The same structure as ``csr_layout``, with 32-bit indices and a configurable tile width. + """ + order = targets.argsort(stable=True) + counts = torch.bincount(targets, minlength=num_nodes) + offsets = torch.cat((counts.new_zeros(1), counts.cumsum(0))) + tile_counts = (counts + edges - 1) // edges + tile_offsets = tile_counts.cumsum(0) - tile_counts + rows = torch.arange(num_nodes, device=counts.device).repeat_interleave(tile_counts) + starts = offsets[rows] + (torch.arange(len(rows), device=rows.device) - tile_offsets[rows]) * edges + hub = tile_counts > hub_tiles + hub_counts = torch.where(hub, tile_counts, 0) + return CSRLayout(*(tensor.to(torch.int32).contiguous() for tensor in ( + offsets, sources[order], types[order], rows, starts, rows[hub[rows]], starts[hub[rows]], + torch.where(hub, hub_counts.cumsum(0) - hub_counts, -1)))) + + +def destination_layout(model): + """Incoming edges of every entity in segments, cached until the graph changes.""" + from ._triton_kgicl import HUB_SEGMENTS, SEGMENT + if model._destination_layout is None: + model._destination_layout = tile_layout(model.edge_index[1], model.edge_index[0], model.edge_type, + model.num_entities, SEGMENT, HUB_SEGMENTS) + return model._destination_layout + + +def fused_scores(model, heads, relations, ids, prompts, edges): + """All-entity scores ``[B, N]`` and first-reach depths (or None) for a query batch.""" + from ._triton_kgicl import Workspace, aggregate, update + from .kgicl import RRELU_LOWER, RRELU_UPPER + + slope = (RRELU_LOWER + RRELU_UPPER) / 2 + device, batch, nodes = heads.device, len(heads), model.num_entities + unique = list(dict.fromkeys(ids)) + position = {q: i for i, q in enumerate(unique)} + tables = model._layer_tables(unique, prompts) + query_index = to_device(torch.tensor([position[q] for q in ids], dtype=torch.int32), device) + layout = destination_layout(model) + space = Workspace(nodes, batch, model.dim, model.attn_dim, device) + rows = torch.arange(batch, device=device) + space.states[0].index_put_((heads, rows), torch.stack([prompts[q][q] for q in ids])) + space.active[0].index_put_((heads, rows), torch.ones((), dtype=torch.int8, device=device)) + depth = None + if model.masked_distances: + depth = torch.full((nodes, batch), -1, dtype=torch.int8, device=device) + depth.index_put_((heads, rows), torch.zeros((), dtype=torch.int8, device=device)) + current = 0 + for i, layer in enumerate(model.gnn_layers): + relation_states, attention_states, query_terms = tables[i] + alpha = layer.w_alpha + aggregate(space, current, relation_states, attention_states, query_terms, query_index, + F.pad(alpha.weight[0], (0, ATTENTION - model.attn_dim)), alpha.bias, layer.Ws_attn.weight, layout, slope) + update(space, current, layer, model.gate, model.layer_norms[i], model.W_final, slope, i == 0, + i == len(model.gnn_layers) - 1) + if depth is not None: + depth.masked_fill_((space.active[1 - current] != 0) & (space.active[current] == 0), i + 1) + current = 1 - current + return space.scores.T.contiguous(), None if depth is None else depth.T.contiguous() diff --git a/dicee/models/_fused_message.py b/dicee/models/_fused_message.py index ff145477c..a44052229 100644 --- a/dicee/models/_fused_message.py +++ b/dicee/models/_fused_message.py @@ -5,9 +5,25 @@ kernel is included in a checkpoint. Triton is imported only on the CUDA path. """ import weakref +from typing import NamedTuple import torch +# Rows with more 32-edge tiles than this are hubs: balanced programs sum their tiles. +HUB_TILES = 8 + + +class CSRLayout(NamedTuple): + """Edges grouped by output row, cut into 32-edge tiles.""" + offsets: torch.Tensor # [N + 1] row pointers + sources: torch.Tensor # input node of each edge, grouped by row + types: torch.Tensor # relation of each edge + tile_rows: torch.Tensor # row of each tile + tile_starts: torch.Tensor # first edge of each tile + hub_rows: torch.Tensor # tile rows/starts restricted to hub rows + hub_starts: torch.Tensor + hub_first: torch.Tensor # [N] index of a hub row's first tile, else -1 + def tensor_version(tensor): return None if tensor.is_inference() else tensor._version @@ -25,7 +41,11 @@ def csr_layout(edge_index, edge_type, num_nodes): tile_offsets = tile_counts.cumsum(0) - tile_counts rows = torch.arange(num_nodes, device=counts.device).repeat_interleave(tile_counts) starts = offsets[rows] + (torch.arange(len(rows), device=rows.device) - tile_offsets[rows]) * 32 - layout = (offsets, edge_index[1, order].contiguous(), edge_type[order].contiguous(), rows, starts) + hub = tile_counts > HUB_TILES + hub_counts = torch.where(hub, tile_counts, 0) + layout = CSRLayout(offsets, edge_index[1, order].contiguous(), edge_type[order].contiguous(), rows, starts, + rows[hub[rows]].contiguous(), starts[hub[rows]].contiguous(), + torch.where(hub, hub_counts.cumsum(0) - hub_counts, -1)) # Inference tensors lack mutation counters. Do not retain a potentially # stale layout for graphs constructed inside inference_mode(). if not edge_index.is_inference() and not edge_type.is_inference(): diff --git a/dicee/models/_inference.py b/dicee/models/_inference.py index 4fa625520..e7feefdab 100644 --- a/dicee/models/_inference.py +++ b/dicee/models/_inference.py @@ -5,19 +5,49 @@ from torch.nn import functional as F +def float32_precision_backends(): + """Precision controls of this PyTorch build, even after cuDNN's RNN module is imported. + + Some PyTorch releases expose ``fp32_precision`` on only a subset of these backends. + """ + if not hasattr(torch.backends.cuda.matmul, 'fp32_precision'): + return () + cudnn, mkldnn = torch.backends.cudnn, torch.backends.mkldnn + # Importing cudnn.rnn shadows the precision object on the module instance. + rnn = getattr(type(cudnn), 'rnn', None) + if rnn is None: + rnn = getattr(cudnn, 'rnn', None) + candidates = (torch.backends, torch.backends.cuda.matmul, cudnn, mkldnn, + getattr(type(cudnn), 'conv', None), rnn, getattr(mkldnn, 'matmul', None)) + return tuple(backend for backend in candidates if hasattr(backend, 'fp32_precision')) + + def float32_precision_token(): """Read precision without mixing PyTorch's legacy and per-backend APIs.""" - if hasattr(torch.backends.cuda.matmul, 'fp32_precision'): - # Include parent settings as well as operator overrides. The legacy - # getters can raise when backends (or cuDNN conv/RNN) use different modes. - cudnn, mkldnn = torch.backends.cudnn, torch.backends.mkldnn - backends = (torch.backends, torch.backends.cuda.matmul, cudnn, mkldnn, - getattr(cudnn, 'conv', None), getattr(cudnn, 'rnn', None), getattr(mkldnn, 'matmul', None)) - # Some PyTorch releases expose fp32_precision on only a subset of these backends. - return tuple(getattr(backend, 'fp32_precision', None) for backend in backends) + backends = float32_precision_backends() + if backends: + return tuple(backend.fp32_precision for backend in backends) return torch.get_float32_matmul_precision(), torch.backends.cudnn.allow_tf32 +def to_device(tensor, device): + """Copy a host tensor to CUDA without a stream synchronization. + + Blocking host-to-device copies wait for all queued GPU work. Staging small + index tensors in pinned memory lets the host continue issuing kernels. + """ + device = torch.device(device) + if device.type != 'cuda' or tensor.device.type != 'cpu': + return tensor.to(device) + return tensor.pin_memory().to(device, non_blocking=True) + + +def host_ids(values): + """Integer IDs as a list, using a host copy attached by the caller when present.""" + ids = getattr(values, '_dicee_ids', None) + return values.tolist() if ids is None else ids + + def inference_only(module): return (not module.training and not torch.is_autocast_enabled(next(module.parameters()).device.type) and (not torch.is_grad_enabled() or not any(p.requires_grad for p in module.parameters()))) @@ -50,6 +80,19 @@ def candidate_slice(candidates, sl): return result +def cached_relation_batch(values, ids): + """Broadcast one cached relation across a same-relation inference batch. + + Beam expansions commonly share a relation. A zero-stride view avoids + copying its full graph representation for every head; consumers must not + mutate this view. Mixed-relation batches retain their original order. + """ + first = values[ids[0]] + if all(key == ids[0] for key in ids): + return first.unsqueeze(0).expand(len(ids), *first.shape) + return torch.stack([values[key] for key in ids]) + + def conditioned_linear(linear, hidden, query): """Apply W[h,q]+b while projecting the shared query only once.""" dim = hidden.shape[-1] diff --git a/dicee/models/_triton_kgicl.py b/dicee/models/_triton_kgicl.py new file mode 100644 index 000000000..b32e123ab --- /dev/null +++ b/dicee/models/_triton_kgicl.py @@ -0,0 +1,271 @@ +"""Fused KG-ICL reasoning: attention-weighted CSR aggregation and GRU node updates. + +States are node-major, ``[entities, queries, dim]``: one program aggregates the +incoming edges of one entity for a fixed block of queries, whose source states +are contiguous. The source term ``Ws_attn h`` of the attention is projected +once per node and layer rather than once per edge. Every (entity, query) row is +computed by a fixed sequence of operations with no atomics, so rows are +deterministic and independent of the query batch: blocks always hold +``QUERIES`` lanes, padded when the batch is smaller, and high in-degree rows +are split into partial sums that are added in order. +""" +import torch +import triton +import triton.language as tl +from triton.language.extra import libdevice + +EDGES = 4 # Edges per chunk; lane j accumulates edges j, j + EDGES, ... of a row in order. +QUERIES = 4 # Queries per aggregation program. +WARPS = 1 # Warps per aggregation program: many small programs hide the gather latency. +SEGMENT = 64 # Edges per partial sum of a hub row. +HUB_SEGMENTS = 4 # Rows with more segments are hubs, split across programs. +ATTENTION = tl.constexpr(16) # Row width of the padded attention tables. + + +@triton.jit +def _alpha(projected, table, query_term, weight, bias, valid, SLOPE: tl.constexpr): + # sigmoid(w_alpha . rrelu(Ws h + Wr r + Wqr q) + b_alpha) over the last axis; padded lanes are zero. + pre = projected + table + query_term + pre = tl.where(pre >= 0, pre, pre * SLOPE) + return tl.where(valid, tl.sigmoid(tl.sum(pre * weight, axis=-1) + bias), 0.) + + +@triton.jit +def _queries(QIDX, QC, WA, B, A: tl.constexpr, AW: tl.constexpr, QUERIES: tl.constexpr): + # The query block of this program: ids, mask, relation slots, attention query terms and w_alpha. + queries = tl.program_id(1) * QUERIES + tl.arange(0, QUERIES) + query_mask = queries < B + lane = tl.arange(0, AW) + qi = tl.load(QIDX + queries, query_mask, other=0).to(tl.int64) + query_term = tl.load(QC + qi[:, None] * ATTENTION + lane[None, :], (lane < A)[None, :], other=0) + return queries, query_mask, qi, query_term, tl.load(WA + lane, lane < A, other=0) + + +@triton.jit +def _segment(H, AS, ACT, TBL, ATBL, SRC, TYPE, start, end, queries, query_mask, qi, query_term, weight, bias, + B, D: tl.constexpr, A: tl.constexpr, AW: tl.constexpr, RS, FEATURES: tl.constexpr, + EDGES: tl.constexpr, QUERIES: tl.constexpr, SLOPE: tl.constexpr): + # Sum of alpha * (h_source + r) over edges [start, end) for a query block, and which queries an edge reaches. + feature = tl.arange(0, FEATURES) + lane = tl.arange(0, AW) + lanes = tl.zeros((EDGES, QUERIES, FEATURES), tl.float32) + reach = tl.zeros((QUERIES,), tl.int32) + for chunk in range(start, end, EDGES): + edge = chunk + tl.arange(0, EDGES) + inside = edge < end + source = tl.load(SRC + edge, inside, other=0).to(tl.int64) + pair = source[:, None] * B + queries[None, :] + present = inside[:, None] & query_mask[None, :] + valid = present & (tl.load(ACT + pair, present, other=0) != 0) + hits = tl.max(valid.to(tl.int32), axis=0) + # Chunks without an active source contribute nothing; skip them. + if tl.max(hits, axis=0) != 0: + slot = qi[None, :] * RS + tl.load(TYPE + edge, inside, other=0).to(tl.int64)[:, None] + attention = valid[:, :, None] & (lane < A)[None, None, :] + projected = tl.load(AS + pair[:, :, None] * AW + lane[None, None, :], attention, other=0) + table = tl.load(ATBL + slot[:, :, None] * ATTENTION + lane[None, None, :], attention, other=0) + alpha = _alpha(projected, table, query_term[None, :, :], weight[None, None, :], bias, valid, SLOPE) + mask = valid[:, :, None] & (feature < D)[None, None, :] + state = tl.load(H + pair[:, :, None] * D + feature[None, None, :], mask, other=0) + rel = tl.load(TBL + slot[:, :, None] * D + feature[None, None, :], mask, other=0) + lanes += alpha[:, :, None] * (state + rel) + reach = tl.maximum(reach, hits) + return tl.sum(lanes, axis=0), reach + + +@triton.jit +def _project(H, ACT, WS, AS, TOTAL, D: tl.constexpr, A: tl.constexpr, AW: tl.constexpr, FEATURES: tl.constexpr, + ROWS: tl.constexpr, PRECISION: tl.constexpr): + # Attention source terms Ws_attn h of active rows; rows that are inactive are never read. + rows = tl.program_id(0).to(tl.int64) * ROWS + tl.arange(0, ROWS) + inside = rows < TOTAL + active = inside & (tl.load(ACT + rows, inside, other=0) != 0) + if tl.max(active.to(tl.int32), axis=0) == 0: + return + feature = tl.arange(0, FEATURES) + lane = tl.arange(0, ATTENTION) + x = tl.load(H + rows[:, None] * D + feature[None, :], active[:, None] & (feature < D)[None, :], other=0) + weight = tl.load(WS + lane[None, :] * D + feature[:, None], (feature < D)[:, None] & (lane < A)[None, :], other=0) + projected = tl.dot(x, weight, input_precision=PRECISION) + tl.store(AS + rows[:, None] * AW + lane[None, :], projected, inside[:, None] & (lane < AW)[None, :]) + + +@triton.jit +def _hub_partials(H, AS, ACT, TBL, ATBL, QC, QIDX, WA, BA, P, PR, PTR, SRC, TYPE, SEGMENT_ROWS, SEGMENT_STARTS, + B, D: tl.constexpr, A: tl.constexpr, AW: tl.constexpr, RS, FEATURES: tl.constexpr, + EDGES: tl.constexpr, QUERIES: tl.constexpr, SEGMENT: tl.constexpr, SLOPE: tl.constexpr): + # One program per segment of a hub row and query block; the row program adds the segments in order. + segment = tl.program_id(0) + queries, query_mask, qi, query_term, weight = _queries(QIDX, QC, WA, B, A, AW, QUERIES) + row = tl.load(SEGMENT_ROWS + segment) + start = tl.load(SEGMENT_STARTS + segment) + end = tl.minimum(start + SEGMENT, tl.load(PTR + row + 1)) + total, reach = _segment(H, AS, ACT, TBL, ATBL, SRC, TYPE, start, end, queries, query_mask, qi, query_term, weight, + tl.load(BA), B, D, A, AW, RS, FEATURES, EDGES, QUERIES, SLOPE) + feature = tl.arange(0, FEATURES) + rows = segment.to(tl.int64) * B + queries + tl.store(P + rows[:, None] * D + feature[None, :], total, query_mask[:, None] & (feature < D)[None, :]) + tl.store(PR + rows, reach, query_mask) + + +@triton.jit +def _aggregate(H, AS, ACT, TBL, ATBL, QC, QIDX, WA, BA, AGG, NEW, PTR, SRC, TYPE, FIRST, P, PR, + B, D: tl.constexpr, A: tl.constexpr, AW: tl.constexpr, RS, FEATURES: tl.constexpr, + EDGES: tl.constexpr, QUERIES: tl.constexpr, SEGMENT: tl.constexpr, SLOPE: tl.constexpr): + row = tl.program_id(0).to(tl.int64) + queries, query_mask, qi, query_term, weight = _queries(QIDX, QC, WA, B, A, AW, QUERIES) + bias = tl.load(BA) + feature = tl.arange(0, FEATURES) + start, end = tl.load(PTR + row), tl.load(PTR + row + 1) + first = tl.load(FIRST + row) + if first < 0: + total, reach = _segment(H, AS, ACT, TBL, ATBL, SRC, TYPE, start, end, queries, query_mask, qi, query_term, weight, + bias, B, D, A, AW, RS, FEATURES, EDGES, QUERIES, SLOPE) + else: + total = tl.zeros((QUERIES, FEATURES), tl.float32) + reach = tl.zeros((QUERIES,), tl.int32) + for segment in range(0, (end - start + SEGMENT - 1) // SEGMENT): + rows = (first + segment).to(tl.int64) * B + queries + total += tl.load(P + rows[:, None] * D + feature[None, :], query_mask[:, None] & (feature < D)[None, :], other=0) + reach = tl.maximum(reach, tl.load(PR + rows, query_mask, other=0)) + current = row * B + queries + active = query_mask & (tl.load(ACT + current, query_mask, other=0) != 0) + # The implicit self-loop edge with relation slot RS - 1. + slot = qi * RS + RS - 1 + lane = tl.arange(0, AW) + attention = active[:, None] & (lane < A)[None, :] + projected = tl.load(AS + current[:, None] * AW + lane[None, :], attention, other=0) + table = tl.load(ATBL + slot[:, None] * ATTENTION + lane[None, :], attention, other=0) + alpha = _alpha(projected, table, query_term, weight[None, :], bias, active, SLOPE) + mask = active[:, None] & (feature < D)[None, :] + state = tl.load(H + current[:, None] * D + feature[None, :], mask, other=0) + rel = tl.load(TBL + slot[:, None] * D + feature[None, :], mask, other=0) + total += alpha[:, None] * (state + rel) + tl.store(AGG + current[:, None] * D + feature[None, :], total, query_mask[:, None] & (feature < D)[None, :]) + tl.store(NEW + current, (active | (reach != 0)).to(tl.int8), query_mask) + + +@triton.jit +def _matrix(W, rows, columns, R: tl.constexpr, C: tl.constexpr): + # W[rows, columns] for an [R, C] row-major matrix, transposed for x @ W.T; padding is zero. + return tl.load(W + columns[None, :] * C + rows[:, None], (rows[:, None] < C) & (columns[None, :] < R), other=0) + + +@triton.jit +def _bias(B, index: tl.constexpr, feature, D: tl.constexpr): + # b[index * D:(index + 1) * D] as one [1, FEATURES] row; padding is zero. + return tl.load(B + index * D + feature, feature < D, other=0)[None, :] + + +@triton.jit +def _gate(x, W, B, index: tl.constexpr, feature, D: tl.constexpr, PRECISION: tl.constexpr): + # One GRU gate block: x @ W[index * D:(index + 1) * D].T + b[index * D:(index + 1) * D]. + value = tl.dot(x, _matrix(W + index * D * D, feature, feature, D, D), input_precision=PRECISION) + return value + _bias(B, index, feature, D) + + +@triton.jit +def _hidden(h, W, B, index: tl.constexpr, feature, D: tl.constexpr, FIRST: tl.constexpr, PRECISION: tl.constexpr): + # The hidden block W_hh h + b_hh of one GRU gate. The first layer's state is zero, so the product is + # exactly zero and only the bias remains. Skipping that dot also avoids a Triton 3.1 compiler abort + # ("DotOperandEncodingAttr non-NvidiaMmaEncodingAttr parent") on tf32x3 dots of a constant zero tensor. + if FIRST: + value = _bias(B, index, feature, D) + else: + value = _gate(h, W, B, index, feature, D, PRECISION) + return value + + +@triton.jit +def _update(AGG, H, NEW, OUT, SCORE, WH, LNW, LNB, WIH, WHH, BIH, BHH, WF, TOTAL, + D: tl.constexpr, FEATURES: tl.constexpr, ROWS: tl.constexpr, EPS: tl.constexpr, SLOPE: tl.constexpr, + FIRST: tl.constexpr, LAST: tl.constexpr, PRECISION: tl.constexpr): + # RReLU(W_h agg) -> LayerNorm -> GRU cell with the previous state; inactive rows stay zero. + # The first layer's GRU state is zero: the query embedding at the head is only a message input. + rows = tl.program_id(0).to(tl.int64) * ROWS + tl.arange(0, ROWS) + inside = rows < TOTAL + active = tl.load(NEW + rows, inside, other=0) != 0 + # Active sets only grow, so an inactive row's outputs are already zero. + if tl.max(active.to(tl.int32), axis=0) == 0: + return + feature = tl.arange(0, FEATURES) + mask = inside[:, None] & (feature[None, :] < D) + offsets = rows[:, None] * D + feature[None, :] + x = tl.load(AGG + offsets, mask & active[:, None], other=0) + if FIRST: + previous = tl.zeros((ROWS, FEATURES), tl.float32) + else: + previous = tl.load(H + offsets, mask, other=0) + y = tl.dot(x, _matrix(WH, feature, feature, D, D), input_precision=PRECISION) + y = tl.where(y >= 0, y, y * SLOPE) + y = tl.where(feature[None, :] < D, y, 0.) + mean = tl.sum(y, axis=1) / D + centered = tl.where(feature[None, :] < D, y - mean[:, None], 0.) + variance = tl.sum(centered * centered, axis=1) / D + norm_weight = tl.load(LNW + feature, feature < D, other=0) + norm_bias = tl.load(LNB + feature, feature < D, other=0) + y = centered * tl.rsqrt(variance + EPS)[:, None] * norm_weight[None, :] + norm_bias[None, :] + reset = tl.sigmoid(_gate(y, WIH, BIH, 0, feature, D, PRECISION) + _hidden(previous, WHH, BHH, 0, feature, D, FIRST, PRECISION)) + update = tl.sigmoid(_gate(y, WIH, BIH, 1, feature, D, PRECISION) + _hidden(previous, WHH, BHH, 1, feature, D, FIRST, PRECISION)) + candidate = libdevice.tanh(_gate(y, WIH, BIH, 2, feature, D, PRECISION) + + reset * _hidden(previous, WHH, BHH, 2, feature, D, FIRST, PRECISION)) + state = (1 - update) * candidate + update * previous + state = tl.where(active[:, None] & (feature[None, :] < D), state, 0.) + if LAST: + weight = tl.load(WF + feature, feature < D, other=0) + tl.store(SCORE + rows, tl.sum(state * weight[None, :], axis=1), inside) + else: + tl.store(OUT + offsets, state, mask) + + +class Workspace: + """Node-major activation buffers of one query batch; outputs of inactive rows stay zero. + + Dense products use 3xTF32 tensor-core dots where available: float32-level + accuracy, and like IEEE FMA dots a fixed per-row operation order. + """ + + def __init__(self, nodes, batch, dim, attention, device): + self.states = [torch.zeros(nodes, batch, dim, device=device) for _ in range(2)] + self.active = [torch.zeros(nodes, batch, dtype=torch.int8, device=device) for _ in range(2)] + self.projected = torch.empty(nodes, batch, max(2, triton.next_power_of_2(attention)), device=device) + self.aggregate = torch.empty(nodes, batch, dim, device=device) + self.scores = torch.zeros(nodes, batch, device=device) + tensor_cores = torch.cuda.get_device_capability(device) >= (8, 0) + self.precision, self.rows = ('tf32x3', 64) if tensor_cores else ('ieee', 32) + + +def _features(dim): + return max(16, triton.next_power_of_2(dim)) + + +def aggregate(space, current, tables, attention_tables, query_constants, query_index, alpha_weight, alpha_bias, + projection, layout, slope): + nodes, batch, dim = space.states[current].shape + attention, width = projection.shape[0], space.projected.shape[-1] + total = nodes * batch + _project[(triton.cdiv(total, space.rows),)](space.states[current], space.active[current], projection, space.projected, + total, dim, attention, width, _features(dim), space.rows, space.precision, + num_warps=4, enable_fp_fusion=False) + shapes = (batch, dim, attention, width, tables.shape[1], _features(dim), EDGES, QUERIES, SEGMENT, slope) + common = (space.states[current], space.projected, space.active[current], tables, attention_tables, query_constants, + query_index, alpha_weight, alpha_bias) + blocks = triton.cdiv(batch, QUERIES) + segments = len(layout.hub_rows) + partials = torch.empty((max(segments, 1), batch, dim), device=tables.device) + reached = torch.empty((max(segments, 1), batch), dtype=torch.int32, device=tables.device) + if segments: + _hub_partials[(segments, blocks)](*common, partials, reached, layout.offsets, layout.sources, layout.types, + layout.hub_rows, layout.hub_starts, *shapes, num_warps=WARPS, enable_fp_fusion=False) + _aggregate[(nodes, blocks)](*common, space.aggregate, space.active[1 - current], layout.offsets, layout.sources, + layout.types, layout.hub_first, partials, reached, *shapes, num_warps=WARPS, enable_fp_fusion=False) + + +def update(space, current, layer, gate, norm, final, slope, first, last): + nodes, batch, dim = space.states[current].shape + total = nodes * batch + _update[(triton.cdiv(total, space.rows),)]( + space.aggregate, space.states[current], space.active[1 - current], space.states[1 - current], space.scores, + layer.W_h.weight, norm.weight, norm.bias, gate.weight_ih_l0, gate.weight_hh_l0, gate.bias_ih_l0, + gate.bias_hh_l0, final.weight, total, dim, _features(dim), space.rows, norm.eps, slope, first, last, + space.precision, num_warps=4, enable_fp_fusion=False) diff --git a/dicee/models/_triton_message.py b/dicee/models/_triton_message.py index 9feab735c..0b3530775 100644 --- a/dicee/models/_triton_message.py +++ b/dicee/models/_triton_message.py @@ -5,70 +5,96 @@ @triton.jit -def _distmult_sum(S, R, B, O, PTR, SRC, TYPE, +def _chunk_sum(S, R, SRC, TYPE, edge, end, batch, feature, + SB: tl.constexpr, SN: tl.constexpr, SD: tl.constexpr, + RB: tl.constexpr, RN: tl.constexpr, RD: tl.constexpr, D: tl.constexpr): + # One 32-edge chunk; the tile shape and warps fix its reduction tree. + source = tl.load(SRC + edge, edge < end, other=0) + relation = tl.load(TYPE + edge, edge < end, other=0) + mask = (edge[:, None] < end) & (feature[None, :] < D) + state = tl.load(S + batch * SB + source[:, None] * SN + feature[None, :] * SD, mask, other=0) + rel = tl.load(R + batch * RB + relation[:, None] * RN + feature[None, :] * RD, mask, other=0) + return tl.sum(state.to(tl.float32) * rel.to(tl.float32), axis=0) + + +@triton.jit +def _tile_sums(S, R, OUT, PTR, SRC, TYPE, ROWS, STARTS, TILES, + SB: tl.constexpr, SN: tl.constexpr, SD: tl.constexpr, + RB: tl.constexpr, RN: tl.constexpr, RD: tl.constexpr, + N: tl.constexpr, D: tl.constexpr, FEATURES: tl.constexpr, EDGES: tl.constexpr, ATOMIC: tl.constexpr): + # One balanced program per tile: add it to its row atomically, or keep it + # for the row program to accumulate in order. + tile, batch = tl.program_id(0), tl.program_id(1) + row = tl.load(ROWS + tile) + end = tl.load(PTR + row + 1) + edge = tl.load(STARTS + tile) + tl.arange(0, EDGES) + feature = tl.arange(0, FEATURES) + total = _chunk_sum(S, R, SRC, TYPE, edge, end, batch, feature, SB, SN, SD, RB, RN, RD, D) + if ATOMIC: + tl.atomic_add(OUT + (batch * N + row) * D + feature, total, feature < D, sem='relaxed') + else: + tl.store(OUT + (batch * TILES + tile) * D + feature, total, feature < D) + + +@triton.jit +def _distmult_sum(S, R, B, O, PTR, SRC, TYPE, FIRST, P, TILES, SB: tl.constexpr, SN: tl.constexpr, SD: tl.constexpr, RB: tl.constexpr, RN: tl.constexpr, RD: tl.constexpr, BB: tl.constexpr, BN: tl.constexpr, BD: tl.constexpr, N: tl.constexpr, D: tl.constexpr, FEATURES: tl.constexpr, - EDGES: tl.constexpr): + EDGES: tl.constexpr, UNROLL: tl.constexpr): row, batch = tl.program_id(0), tl.program_id(1) feature = tl.arange(0, FEATURES) start, end = tl.load(PTR + row), tl.load(PTR + row + 1) + first = tl.load(FIRST + row) total = tl.full((FEATURES,), 0, tl.float32) - for first in range(start, end, EDGES): - edge = first + tl.arange(0, EDGES) - source = tl.load(SRC + edge, edge < end, other=0) - relation = tl.load(TYPE + edge, edge < end, other=0) - mask = (edge[:, None] < end) & (feature[None, :] < D) - state = tl.load(S + batch * SB + source[:, None] * SN + feature[None, :] * SD, mask, other=0) - rel = tl.load(R + batch * RB + relation[:, None] * RN + feature[None, :] * RD, mask, other=0) - total += tl.sum(state.to(tl.float32) * rel.to(tl.float32), axis=0) + if first < 0: + for chunk in range(start, end, EDGES): + edge = chunk + tl.arange(0, EDGES) + total += _chunk_sum(S, R, SRC, TYPE, edge, end, batch, feature, SB, SN, SD, RB, RN, RD, D) + else: + # Same chunk sums, accumulated in the same order as the inline loop. + count = (end - start + EDGES - 1) // EDGES + partial = P + (batch * TILES + first) * D + feature + for block in range(0, count, UNROLL): + for offset in tl.static_range(UNROLL): + valid = block + offset < count + value = tl.load(partial + (block + offset) * D, (feature < D) & valid, other=0) + total = tl.where(valid, total + value, total) boundary = tl.load(B + batch * BB + row * BN + feature * BD, feature < D, other=0) tl.store(O + (batch * N + row) * D + feature, total + boundary, feature < D) -@triton.jit -def _distmult_tiles(S, R, O, PTR, SRC, TYPE, ROWS, STARTS, - SB: tl.constexpr, SN: tl.constexpr, SD: tl.constexpr, - RB: tl.constexpr, RN: tl.constexpr, RD: tl.constexpr, - N: tl.constexpr, D: tl.constexpr, FEATURES: tl.constexpr): - tile, batch = tl.program_id(0), tl.program_id(1) - row = tl.load(ROWS + tile) - edge = tl.load(STARTS + tile) + tl.arange(0, 32) - end = tl.load(PTR + row + 1) - feature = tl.arange(0, FEATURES) - source = tl.load(SRC + edge, edge < end, other=0) - relation = tl.load(TYPE + edge, edge < end, other=0) - mask = (edge[:, None] < end) & (feature[None, :] < D) - state = tl.load(S + batch * SB + source[:, None] * SN + feature[None, :] * SD, mask, other=0) - rel = tl.load(R + batch * RB + relation[:, None] * RN + feature[None, :] * RD, mask, other=0) - total = tl.sum(state.to(tl.float32) * rel.to(tl.float32), axis=0) - tl.atomic_add(O + (batch * N + row) * D + feature, total, feature < D, sem='relaxed') - - def distmult_sum(states, boundary, relations, layout): batch, nodes, dim = states.shape + edges = (layout.offsets, layout.sources, layout.types) + shape = (*states.stride(), *relations.stride()) + width = triton.next_power_of_2(dim) # Split high-degree rows into independent tiles. This avoids serial hub # loops and retains O(B*N*D) activation memory. Deterministic mode instead - # assigns each output row to one program, without floating-point atomics. - if len(layout[1]) > 16 * nodes and not torch.are_deterministic_algorithms_enabled(): + # keeps each row's sequential tile order, without floating-point atomics. + if len(layout.sources) > 16 * nodes and not torch.are_deterministic_algorithms_enabled(): # Tile atomics always accumulate in float32, including FP16/BF16 input. output = boundary.to(torch.float32).clone(memory_format=torch.contiguous_format) - if batch and len(layout[3]): - _distmult_tiles[(len(layout[3]), batch)]( - states, relations, output, *layout, - *states.stride(), *relations.stride(), nodes, dim, - triton.next_power_of_2(dim), num_warps=4, enable_fp_fusion=False, - ) + if batch and len(layout.tile_rows): + _tile_sums[(len(layout.tile_rows), batch)]( + states, relations, output, *edges, layout.tile_rows, layout.tile_starts, 0, + *shape, nodes, dim, width, 32, True, num_warps=4, enable_fp_fusion=False) return output.to(states.dtype) output = torch.empty_like(states, memory_format=torch.contiguous_format) - if batch and nodes: - _distmult_sum[(nodes, batch)]( - states, relations, boundary, output, *layout[:3], - *states.stride(), *relations.stride(), *boundary.stride(), - nodes, dim, triton.next_power_of_2(dim), 32, - num_warps=4, enable_fp_fusion=False, - ) + if not batch or not nodes: + return output + # Hub rows read their tile sums from balanced programs instead of looping + # serially; adding them in order is bitwise equal to the single-pass loop. + hubs = len(layout.hub_rows) + partials = torch.empty((batch, max(hubs, 1), dim), device=states.device, dtype=torch.float32) + if hubs: + _tile_sums[(hubs, batch)]( + states, relations, partials, *edges, layout.hub_rows, layout.hub_starts, hubs, + *shape, nodes, dim, width, 32, False, num_warps=4, enable_fp_fusion=False) + _distmult_sum[(nodes, batch)]( + states, relations, boundary, output, *edges, layout.hub_first, partials, max(hubs, 1), + *shape, *boundary.stride(), nodes, dim, width, 32, 8, num_warps=4, enable_fp_fusion=False) return output diff --git a/dicee/models/flock.py b/dicee/models/flock.py index a1e1b25a9..d6d1cdcf3 100644 --- a/dicee/models/flock.py +++ b/dicee/models/flock.py @@ -316,6 +316,60 @@ class Flock(FlockBase): """Entity-prediction Flock with official ``flock_entity.pth`` parameters.""" name = "Flock" + @torch.no_grad() + def forward_k_vs_all_seeded(self, queries, seeds, *, samples=None): + """Batch inference with independent, batch-invariant sampling seeds.""" + if self.training: + raise ValueError('Seeded scoring requires evaluation mode') + n, _ = self._require_graph() + samples = self.test_samples if samples is None else samples + if type(samples) is not int or samples < 1 or len(seeds) != len(queries): + raise ValueError('Provide a seed per query and a positive sample count') + if any(type(seed) is not int or not 0 <= seed < 2**63 for seed in seeds): + raise ValueError('Seeds must be integers in [0, 2**63)') + queries = torch.as_tensor(queries, device=self.device, dtype=torch.long) + if queries.ndim != 2 or queries.shape[1] != 2: + raise ValueError('Expected [batch, 2] head/relation queries') + if not len(queries): + return self.node_init.new_empty((0, n)) + if ((queries[:, 0] < 0) | (queries[:, 0] >= n) | (queries[:, 1] < 0) | (queries[:, 1] >= self.num_relations)).any(): + raise ValueError('Query outside the public vocabulary') + heads = queries[:, 0].repeat_interleave(samples) + relations = self.relation_id_map[queries[:, 1]].repeat_interleave(samples) + cpu_heads = heads.cpu() + generators = {} + + def draw(start): + records = [] + for index in range(start, min(start + self.query_batch_size, len(heads))): + row = index // samples + if row not in generators: + generators[row] = torch.Generator().manual_seed(seeds[row]) + records.append(self._draw_walks(self._walk_graph, cpu_heads[index:index + 1], None, generators[row])) + if index % samples == samples - 1: + del generators[row] + packed = tuple(torch.cat(parts, dim=1) for parts in zip(*records)) + return self._transfer_records(packed) if self.device.type == 'cuda' else packed + + def score(start, records): + stop = min(start + self.query_batch_size, len(heads)) + candidates = torch.arange(n, device=self.device).expand(stop - start, -1) + return self.score_walks(heads[start:stop], relations[start:stop], candidates, records) + + starts = range(0, len(heads), self.query_batch_size) + if self.prefetch_walks and self.device.type == 'cuda' and len(heads) > self.query_batch_size: + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(draw, 0) + output = [] + for start in starts: + records = future.result() + if start + self.query_batch_size < len(heads): + future = pool.submit(draw, start + self.query_batch_size) + output.append(score(start, records)) + else: + output = [score(start, draw(start)) for start in starts] + return torch.cat(output).view(len(queries), samples, n).mean(1) + def _score(self, heads, relations, candidates, query_relations, edges): # Unlike ULTRA/TRIX, query conditioning occurs after head-to-tail # conversion, using the actual (possibly inverse) relation ID. diff --git a/dicee/models/graph_model.py b/dicee/models/graph_model.py index 16aa817b3..dac05051d 100644 --- a/dicee/models/graph_model.py +++ b/dicee/models/graph_model.py @@ -34,7 +34,7 @@ from torch import nn from ._fused_message import tensor_version -from ._inference import float32_precision_token +from ._inference import float32_precision_token, to_device from .base_model import BaseKGE @@ -61,6 +61,9 @@ def __init__(self, args): for name in ('graph_triples', 'relation_id_map', 'edge_index', 'edge_type'): self.register_buffer(name, None, persistent=False) self.num_direct_relations = 0 + # Which relation conditions relation reasoning for a tail query (h, r, ?): 'direct', as upstream link + # prediction does (an inverse query uses its direct relation), or 'query', the queried relation itself. + self.relation_conditioning = 'direct' def clear_inference_cache(self): """Drop derived representations when graph, weights or device change.""" @@ -79,7 +82,7 @@ def inference_token(self): tensors = list(self.parameters()) + list(self.buffers()) if any(t.is_inference() for t in tensors): return None - return (getattr(self, '_inference_backend', 'auto'), torch.is_autocast_enabled(self.device.type), + return (getattr(self, '_inference_backend', 'auto'), self.relation_conditioning, torch.is_autocast_enabled(self.device.type), torch.are_deterministic_algorithms_enabled(), float32_precision_token(), tuple(getattr(m, 'inference_compile', False) for m in self.modules() if hasattr(m, 'inference_backend')), @@ -254,7 +257,14 @@ def forward_k_vs_sample(self, x, target_entity_idx): raise ValueError('Tail queries must have shape [B, 2] in (head, relation) order') if x.numel() and (x.min() < 0 or x[:, 0].max() >= self.num_entities or x[:, 1].max() >= self.num_relations): raise ValueError('Query IDs are outside the graph vocabulary') - x = x.to(device=self.device) + # Host queries were validated without a device read; keep their relation + # IDs on the host so inference caches need no device-to-host copy. + host_ids = None + direct = self.relation_conditioning == 'direct' + if x.device.type == 'cpu' and self.device.type != 'cpu': + mapped = self._host_relation_map()[x[:, 1]] + host_ids = (mapped % self.num_direct_relations if direct else mapped).tolist() + x = to_device(x, self.device) queries = torch.stack((x[:, 0], self.relation_id_map[x[:, 1]]), dim=1) candidates = target_entity_idx.to(dtype=torch.long) if candidates.ndim == 1: @@ -267,7 +277,20 @@ def forward_k_vs_sample(self, x, target_entity_idx): candidates = candidates.to(device=self.device) if all_entities: candidates._dicee_all_entities = True - return self._score(queries[:, 0], queries[:, 1], candidates, queries[:, 1] % self.num_direct_relations, self._training_edges(queries=queries)) + query_relations = queries[:, 1] % self.num_direct_relations if direct else queries[:, 1] + if host_ids is not None: + query_relations._dicee_ids = host_ids + return self._score(queries[:, 0], queries[:, 1], candidates, query_relations, self._training_edges(queries=queries)) + + def _host_relation_map(self): + """Host copy of the internal relation map, reused while the buffer is unchanged.""" + mapping = self.relation_id_map + version = tensor_version(mapping) + cached = getattr(self, '_host_relation_cache', None) + if cached is None or cached[0] is not mapping or cached[1] != version or version is None: + cached = mapping, version, mapping.cpu() + self._host_relation_cache = cached + return cached[2] def forward_k_vs_all(self, x): num_entities, _ = self._require_graph() diff --git a/dicee/models/kgicl.py b/dicee/models/kgicl.py new file mode 100644 index 000000000..234053f2f --- /dev/null +++ b/dicee/models/kgicl.py @@ -0,0 +1,463 @@ +"""Pure PyTorch KG-ICL, compatible with the official nju-websoft/KG-ICL checkpoints. + +Reimplementation of Cui et al., "A Prompt-Based Knowledge Graph Foundation +Model for Universal In-Context Reasoning" (NeurIPS 2024, +https://arxiv.org/abs/2410.12288), verified against +https://github.com/nju-websoft/KG-ICL at UPSTREAM_COMMIT. Module names follow +the released ``model_best.tar`` state dictionaries. Prompt graphs are runtime +context derived from the attached graph (see kgicl_prompts.py), never part of +the transferable state. docs/kgicl.md documents the corrected upstream defects. +""" +import hashlib +from collections import OrderedDict +from typing import Optional, Sequence, cast + +import torch +from torch import nn +from torch.nn import functional as F + +from ._fused_kgicl import ATTENTION, fused_reasoning_supported, fused_scores, layer_tables +from ._inference import candidate_features, candidate_slice, host_ids, inference_only +from .graph_model import GraphKGE +from .kgicl_prompts import PromptGraph, PromptSampler + +UPSTREAM_COMMIT = "6a3166e347ae468acdfb30a70a2cf3608b66b8f1" +# nn.RReLU's default bounds; in evaluation its negative slope is their mean. +RRELU_LOWER, RRELU_UPPER = 1. / 8, 1. / 3 + + +def _split_linear(linear, *parts: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor: + """``linear(cat([table[index] for table, index in parts], -1))`` without the concatenation. + + Each table is projected by its block of the weight before the per-edge gather. + """ + dim = parts[0][0].shape[-1] + value = F.linear(parts[0][0], linear.weight[:, :dim])[parts[0][1]] + for k, (table, index) in enumerate(parts[1:], 1): + value = value.add_(F.linear(table, linear.weight[:, k * dim:(k + 1) * dim])[index]) + return value.add_(linear.bias) + + +class PromptEncoder(nn.Module): + """Upstream ``PromptEncoder`` with the released configuration. + + Settings: unified tokens, concatenated entity messages, sigmoid attention, + max aggregation for entities and relations. Relation tokens start at zero + except the query relation. ``ent_norm`` multiplies messages twice, as in + the released model, so the checkpoints keep their trained behavior. + """ + + def __init__(self, dim=32, num_layers=3, hops=3): + super().__init__() + self.dim, self.num_layers, self.hops = dim, num_layers, hops + self.start_relation_embeddings = nn.Embedding(1, dim) + self.position_embedding = nn.Embedding((hops + 1) ** 2, dim) + self.self_loop_embedding = nn.Embedding(1, dim) + for embedding in (self.start_relation_embeddings, self.position_embedding, self.self_loop_embedding): + nn.init.xavier_normal_(embedding.weight.data) + self.act = nn.RReLU() + self.W_ht2r = nn.ModuleList([nn.Linear(3 * dim, dim) for _ in range(num_layers)]) + self.W_message = nn.ModuleList([nn.Linear(3 * dim, dim) for _ in range(num_layers)]) + self.alpha = nn.ModuleList([nn.Linear(2 * dim, 1) for _ in range(num_layers)]) + self.beta = nn.ModuleList([nn.Linear(2 * dim, 1) for _ in range(num_layers)]) + self.loop_transfer = nn.ModuleList([nn.Linear(2 * dim, dim) for _ in range(num_layers)]) + self.ent_transfer = nn.ModuleList([nn.Linear(dim, dim) for _ in range(num_layers)]) + self.rel_transfer = nn.ModuleList([nn.Linear(dim, dim) for _ in range(num_layers)]) + self.final_to_rel_embeddings = nn.Linear(dim * num_layers, dim) + self.dropout = nn.Dropout(0.3) + self.layer_norm_rels = nn.ModuleList([nn.LayerNorm(dim) for _ in range(num_layers + 1)]) + self.layer_norm_ents = nn.ModuleList([nn.LayerNorm(dim) for _ in range(num_layers + 1)]) + self.layer_norm_loop = nn.ModuleList([nn.LayerNorm(dim) for _ in range(num_layers + 1)]) + + @staticmethod + def _max(values, index, size): + # torch_scatter.scatter_max semantics: empty groups are zero. + out = values.new_zeros(size, values.shape[-1]) + return out.scatter_reduce(0, index[:, None].expand_as(values), values, 'amax', include_self=False) + + def forward(self, graphs: Sequence[PromptGraph], relation: int, num_relations: int) -> torch.Tensor: + """Mean prompt representation ``[num_relations + 1, dim]``; the last row is the self-loop relation.""" + device = self.position_embedding.weight.device + shots = len(graphs) + sizes = torch.tensor([g.num_nodes for g in graphs]) + node_offsets = torch.cat((sizes.new_zeros(1), sizes.cumsum(0)[:-1])) + num_nodes = int(sizes.sum()) + edge_index = torch.cat([g.edge_index + offset for g, offset in zip(graphs, node_offsets.tolist())], 1).to(device) + slots = torch.arange(shots) * num_relations + edge_type = torch.cat([g.edge_type + slot for g, slot in zip(graphs, slots.tolist())]).to(device) + edge_query = torch.cat([torch.full((g.edge_type.numel(),), relation + slot) for g, slot in zip(graphs, slots.tolist())]).to(device) + labels = torch.cat([g.labels for g in graphs]).to(device) + heads = (node_offsets + torch.tensor([g.head for g in graphs])).to(device) + tails = (node_offsets + torch.tensor([g.tail for g in graphs])).to(device) + query_slots = (relation + slots).to(device) + + node = self.position_embedding.weight[labels[:, 0] * (self.hops + 1) + labels[:, 1]] + node = node.index_put((heads,), self.position_embedding.weight[0].expand(shots, -1)) + node = node.index_put((tails,), self.position_embedding.weight[1].expand(shots, -1)) + rel = node.new_zeros(num_relations * shots, self.dim) + rel = rel.index_put((query_slots,), self.start_relation_embeddings.weight[0].expand(shots, -1)) + loop = self.self_loop_embedding.weight.expand(shots, -1) + source, target = edge_index + degree = torch.zeros(num_nodes, device=device).index_add_(0, target, torch.ones_like(target, dtype=node.dtype)) + inverse_sqrt = degree.pow(-0.5) + inverse_sqrt = inverse_sqrt.masked_fill(inverse_sqrt == float('inf'), 0) + norm = (inverse_sqrt[target] * inverse_sqrt[source])[:, None] + layers = [] + for i in range(self.num_layers): + if self.training: + # RReLU samples a slope per element, so training keeps upstream's per-edge inputs. + r, q = rel[edge_type], rel[edge_query] + message = self.act(self.W_message[i](torch.cat((node[source], r, q), -1))) + alpha = torch.sigmoid(self.alpha[i](self.act(torch.cat((r, q), -1)))) + message = message * alpha * norm * norm + else: + # Project entity and relation tables before gathering them per edge; this + # avoids [edges, 3 * dim] inputs for the hub-sized prompt graphs of large KGs. + message = self.act(_split_linear(self.W_message[i], (node, source), (rel, edge_type), (rel, edge_query))) + active = self.act(rel) + alpha = torch.sigmoid(_split_linear(self.alpha[i], (active, edge_type), (active, edge_query))) + message = message.mul_(alpha).mul_(norm).mul_(norm) + node = self.act(self.ent_transfer[i](self._max(message, target, num_nodes))) + node = self.layer_norm_ents[i](node) + if self.training: + message = self.act(self.W_ht2r[i](torch.cat((node[source], node[target], q), -1))) + message = message * torch.sigmoid(self.beta[i](torch.cat((r, q), -1))) + else: + message = self.act(_split_linear(self.W_ht2r[i], (node, source), (node, target), (rel, edge_query))) + message = message.mul_(torch.sigmoid(_split_linear(self.beta[i], (rel, edge_type), (rel, edge_query)))) + rel = self.act(self.rel_transfer[i](self._max(message, edge_type, len(rel)))) + rel + rel = self.layer_norm_rels[i](rel) + layers.append(rel) + loop = loop + self.act(self.loop_transfer[i](torch.cat((loop, rel[query_slots]), -1))) + loop = self.layer_norm_loop[i](loop) + final = self.layer_norm_rels[-1](self.act(self.final_to_rel_embeddings(torch.cat(layers, -1)))) + final = self.dropout(final.view(shots, num_relations, self.dim).mean(0)) + return torch.cat((final, loop.mean(0, keepdim=True))) + + +class _UnusedConv(nn.Module): + """Parameters of the official checkpoints' unused NBFNet layer, kept for strict loading.""" + + def __init__(self, dim): + super().__init__() + self.linear = nn.Linear(2 * dim, dim) + self.relation_projection = nn.Sequential(nn.Linear(dim, dim), nn.ReLU(), nn.Linear(dim, dim)) + + +class GNNLayer(nn.Module): + """Query-aware attention with TransE-style ``h + r`` messages and sum aggregation.""" + + def __init__(self, dim=32, attn_dim=5): + super().__init__() + self.inference_backend = 'auto' + self.Ws_attn = nn.Linear(dim, attn_dim, bias=False) + self.Wr_attn = nn.Linear(dim, attn_dim, bias=False) + self.Wqr_attn = nn.Linear(dim, attn_dim) + self.w_alpha = nn.Linear(attn_dim, 1) + self.W_h = nn.Linear(dim, dim, bias=False) + self.conv = _UnusedConv(dim) + self.act = nn.RReLU() + + def forward(self, hidden, source, edge_batch, edge_relation, target, num_targets, relations, query): + hs = hidden[source] + hr = relations[edge_batch, edge_relation] + hq = relations[edge_batch, query[edge_batch]] + # Upstream constructs a fresh nn.RReLU per call, which always samples + # random slopes. Evaluation uses the module's deterministic mean slope. + attention = F.rrelu(self.Ws_attn(hs) + self.Wr_attn(hr) + self.Wqr_attn(hq), RRELU_LOWER, RRELU_UPPER, self.training) + alpha = torch.sigmoid(self.w_alpha(attention)) + message = alpha * (hs + hr) + aggregate = hidden.new_zeros(num_targets, hidden.shape[-1]).index_add_(0, target, message) + return self.act(self.W_h(aggregate)) + + +class KGICL(GraphKGE): + """KG-ICL entity prediction with the official state dictionary and DICE triple order. + + Prompt graphs for a query relation are drawn deterministically from the + attached graph; their encodings are cached per relation. The reasoner + expands one hop per layer from the query head, as upstream does: entities + outside the reached set have no state and score zero. + """ + + name = 'KGICL' + config_prefix = 'kgicl' + graph_filename = 'kgicl_graph.pt' + checkpoint_hint = ('official KG-ICL-6L requires kgicl_dim=32, kgicl_attn_dim=5, kgicl_num_layers=6, ' + 'kgicl_prompt_layers=3, kgicl_prompt_hops=3; all keys and shapes must match') + deterministic_inference = True + + def __init__(self, args): + super().__init__(args) + self.dim = args.get('kgicl_dim', 32) + self.attn_dim = args.get('kgicl_attn_dim', 5) + self.num_layers = args.get('kgicl_num_layers', 6) + self.prompt_layers = args.get('kgicl_prompt_layers', 3) + self.prompt_hops = args.get('kgicl_prompt_hops', 3) + self.shots = args.get('kgicl_shots', 5) + self.open_nodes = args.get('kgicl_prompt_open_nodes', 50) + self.prompt_seed = args.get('kgicl_prompt_seed', 0) + masked = args.get('kgicl_masked_distances') + self.masked_distances = tuple(sorted(set(int(d) for d in masked))) if masked else () + if min(self.dim, self.attn_dim, self.num_layers, self.prompt_layers, self.prompt_hops, self.shots) < 1: + raise ValueError('KG-ICL dimensions, layers, hops and shots must be positive') + if self.open_nodes < 0 or any(d < 0 for d in self.masked_distances): + raise ValueError('KG-ICL open nodes and masked distances must be nonnegative') + if not isinstance(self.prompt_seed, int) or not 0 <= self.prompt_seed < 2 ** 63: + raise ValueError('kgicl_prompt_seed must be a nonnegative integer less than 2**63') + self.relation_encoder = PromptEncoder(self.dim, self.prompt_layers, self.prompt_hops) + self.gnn_layers = nn.ModuleList([GNNLayer(self.dim, self.attn_dim) for _ in range(self.num_layers)]) + self.rel_transfer = nn.ModuleList([nn.Linear(self.dim, self.dim) for _ in range(self.num_layers)]) + self.layer_norms = nn.ModuleList([nn.LayerNorm(self.dim) for _ in range(self.num_layers)]) + self.layer_norms_rel = nn.ModuleList([nn.LayerNorm(self.dim) for _ in range(self.num_layers)]) + # Unused by the released model's forward pass; retained for strict checkpoint loading. + self.layer_norms_query = nn.ModuleList([nn.LayerNorm(self.dim) for _ in range(self.num_layers)]) + self.query_transfer = nn.ModuleList([nn.Linear(self.dim, self.dim) for _ in range(self.num_layers)]) + self.W_score = nn.ModuleList([nn.Linear(2 * self.dim, 1, bias=False) for _ in range(self.num_layers)]) + self.dropout = nn.Dropout(args.get('kgicl_dropout', 0.0)) + self.W_final = nn.Linear(self.dim, 1, bias=False) + self.gate = nn.GRU(self.dim, self.dim) + # Prompt encodings and the fused path's per-layer relation tables, like ULTRA's caches. + self.relation_cache_mb = args.get('graph_relation_cache_mb', 64) + self.projection_cache_mb = args.get('graph_projection_cache_mb', 64) + if min(self.relation_cache_mb, self.projection_cache_mb) < 0: + raise ValueError('Graph cache limits must be nonnegative') + self._prompt_overrides: dict[int, list[PromptGraph]] = {} + self._sampler: Optional[PromptSampler] = None + self.set_inference_backend(args.get('graph_inference_backend', 'auto')) + + # ----- checkpoint, graph and cache lifecycle ----- + + def load_pretrained(self, path): + """Load an official ``model_best.tar`` (``state_dict`` key), a ``model`` wrapper or a bare state.""" + state = torch.load(path, map_location='cpu', weights_only=True) + if 'state_dict' in state and isinstance(state['state_dict'], dict): + state = state['state_dict'] + state = state.get('model', state) + expected = self.state_dict() + if set(state) != set(expected) or any(state[k].shape != expected[k].shape for k in expected): + raise ValueError(f'Checkpoint architecture mismatch for {self.name}: {self.checkpoint_hint}') + self.load_state_dict(state, strict=True) + return self + + def clear_inference_cache(self): + self._prompt_cache = OrderedDict() + self._prompt_cache_token = None + self._table_cache = OrderedDict() + self._table_cache_token = None + self._layout = None + self._destination_layout = None + + def _build_relation_graph(self): + # KG-ICL samples prompt graphs lazily; no relation graph is constructed. + self.clear_inference_cache() + self._sampler = None + self._prompt_overrides = {} + + def prompt_settings(self): + """Settings that determine prompt graphs and score masks, for cache identities.""" + return (self.shots, self.prompt_hops, self.open_nodes, self.prompt_seed, self.masked_distances, + tuple(sorted((q, id(g)) for q, g in self._prompt_overrides.items()))) + + def inference_token(self): + token = super().inference_token() + return None if token is None else (*token, self.prompt_settings()) + + def prompt_identity(self) -> dict: + """Stable description of prompt sampling for persistent score caches. + + Replayed prompts are identified by a digest of their tensors. + """ + digest = None + if self._prompt_overrides: + content = hashlib.sha256() + for relation in sorted(self._prompt_overrides): + for graph in self._prompt_overrides[relation]: + content.update(repr((relation, graph.head, graph.tail)).encode()) + for tensor in (graph.edge_index, graph.edge_type, graph.labels): + content.update(tensor.detach().cpu().contiguous().numpy().tobytes()) + digest = content.hexdigest() + return dict(sampler='kgicl-prompt-v1', shots=self.shots, hops=self.prompt_hops, open_nodes=self.open_nodes, + seed=self.prompt_seed, masked_distances=list(self.masked_distances), replayed=digest) + + @property + def _entities(self) -> int: + """Entity count of the attached graph; scoring and sampling run only after ``set_graph``.""" + return cast(int, self.num_entities) + + @property + def sampler(self) -> PromptSampler: + """Prompt sampler of the attached graph, created on first use.""" + self._require_graph() + if self._sampler is None: + mapping = self.relation_id_map.detach().cpu().tolist() + public = {internal: external for external, internal in enumerate(mapping)} + self._sampler = PromptSampler(self.graph_triples, self._entities, self.num_direct_relations, + [public[r] for r in range(self.num_direct_relations)], shots=self.shots, + hops=self.prompt_hops, open_nodes=self.open_nodes, seed=self.prompt_seed) + return self._sampler + + def use_prompts(self, prompts: Optional[dict] = None): + """Replay fixed prompt graphs, keyed by internal query relation; ``None`` restores sampling. + + Each value lists one ``PromptGraph`` per shot. Attaching a graph clears the replay. + """ + prompts = dict(prompts or {}) + if any(len(graphs) < 1 for graphs in prompts.values()): + raise ValueError('Replayed prompts need at least one graph per relation') + self._prompt_overrides = {int(q): list(graphs) for q, graphs in prompts.items()} + self.clear_inference_cache() + return self + + def prompt_graphs(self, relation: int) -> list: + """Prompt graphs of an internal query relation; training draws examples with replacement.""" + if relation in self._prompt_overrides: + return self._prompt_overrides[relation] + sampler = self.sampler + slots = None + if self.training: + # Upstream training picks one of the loaded examples at random for each shot. + count = len(sampler.examples(relation % self.num_direct_relations)) + if count: + slots = torch.randint(count, (self.shots,)).tolist() + return sampler.prompts(relation, slots) + + def encode_prompts(self, relation: int, graphs: Optional[Sequence[PromptGraph]] = None) -> torch.Tensor: + """Prompt representation ``[2R + 1, dim]`` of an internal query relation.""" + self._require_graph() + graphs = self.prompt_graphs(relation) if graphs is None else graphs + encoded: torch.Tensor = self.relation_encoder(graphs, relation, 2 * self.num_direct_relations) + return encoded + + def _prompts(self, relations): + """Per-query prompt representations, cached by relation during inference.""" + ids = host_ids(relations) + token = self.inference_token() if inference_only(self) and self.relation_cache_mb else None + if token is None: + values = {q: self.encode_prompts(q) for q in dict.fromkeys(ids)} + return values, ids + if token != self._prompt_cache_token: + self._prompt_cache.clear() + self._prompt_cache_token = token + capacity = int(self.relation_cache_mb * 2**20) // ((2 * self.num_direct_relations + 1) * self.dim + * next(self.parameters()).element_size()) + values = {} + for q in dict.fromkeys(ids): + if q not in self._prompt_cache: + self._prompt_cache[q] = self.encode_prompts(q).detach() + self._prompt_cache.move_to_end(q) + values[q] = self._prompt_cache[q] + # Keep current-call values alive even when they exceed the LRU capacity. + while len(self._prompt_cache) > max(capacity, 1): + self._prompt_cache.popitem(last=False) + return values, ids + + def _layer_tables(self, relations, prompts): + """Stacked per-layer relation tables of the fused path, cached per relation like prompts.""" + token = self._prompt_cache_token + size = (self.num_layers * (2 * self.num_direct_relations + 1) * (self.dim + ATTENTION) + * next(self.parameters()).element_size()) + capacity = int(self.projection_cache_mb * 2**20) // size + if token is None or not capacity: + values = {q: layer_tables(self, q, prompts[q]) for q in relations} + else: + if token != self._table_cache_token: + self._table_cache.clear() + self._table_cache_token = token + values = {} + for q in relations: + if q not in self._table_cache: + self._table_cache[q] = layer_tables(self, q, prompts[q]) + self._table_cache.move_to_end(q) + values[q] = self._table_cache[q] + while len(self._table_cache) > max(capacity, 1): + self._table_cache.popitem(last=False) + return [tuple(torch.stack([values[q][i][k] for q in relations]) for k in range(3)) + for i in range(self.num_layers)] + + # ----- reasoning ----- + + def _out_edges(self, edges): + """Edges grouped by source entity: offsets [N + 1], targets, types.""" + cached = self._layout + if cached is not None and cached[0] is edges[0] and cached[1] is edges[1]: + return cached[2] + edge_index, edge_type = edges + order = edge_index[0].argsort(stable=True) + counts = torch.bincount(edge_index[0], minlength=self._entities) + layout = (torch.cat((counts.new_zeros(1), counts.cumsum(0))), edge_index[1, order], edge_type[order]) + if not self.training: + self._layout = (edge_index, edge_type, layout) + return layout + + def _expand_scores(self, heads, relations, prompts, edges): + """Upstream's hop-by-hop reasoning for a batch; returns ``[B, N]`` scores and first-reach depths.""" + n, dim = self._entities, self.dim + offsets, targets, types = self._out_edges(edges) + self_loop = 2 * self.num_direct_relations + batch = torch.arange(len(heads), device=heads.device) + node_batch, node_entity = batch, heads + hidden = prompts[batch, relations] + rel, state = prompts, None + depth = heads.new_zeros(len(heads)) + for i, layer in enumerate(self.gnn_layers): + counts = offsets[node_entity + 1] - offsets[node_entity] + source = torch.arange(len(node_entity), device=heads.device).repeat_interleave(counts) + starts = offsets[node_entity].repeat_interleave(counts) + position = starts + torch.arange(len(source), device=heads.device) - (counts.cumsum(0) - counts).repeat_interleave(counts) + loops = torch.arange(len(node_entity), device=heads.device) + source = torch.cat((source, loops)) + edge_relation = torch.cat((types[position], torch.full_like(loops, self_loop))) + edge_target = torch.cat((targets[position], node_entity)) + edge_batch = node_batch[source] + keys, target = (edge_batch * n + edge_target).unique(sorted=True, return_inverse=True) + previous = torch.searchsorted(keys, node_batch * n + node_entity) + rel = self.layer_norms_rel[i](rel + F.relu(self.rel_transfer[i](rel))) + hidden = layer(hidden, source, edge_batch, edge_relation, target, len(keys), rel, relations) + initial = hidden.new_zeros(len(keys), dim) + if state is not None: + initial = initial.index_copy(0, previous, state) + hidden = self.layer_norms[i](self.dropout(hidden)) + hidden, state = self.gate(hidden.unsqueeze(0), initial.unsqueeze(0)) + hidden, state = hidden.squeeze(0), state.squeeze(0) + rel = self.layer_norms_rel[i](rel) + depth = torch.full_like(keys, i + 1).index_copy(0, previous, depth) + node_batch, node_entity = keys // n, keys % n + scores = hidden.new_zeros(len(heads), n).index_put((node_batch, node_entity), self.W_final(hidden).squeeze(-1)) + reached = torch.full((len(heads), n), -1, dtype=torch.long, device=heads.device) + reached = reached.index_put((node_batch, node_entity), depth) + return scores, reached + + def _mask_distances(self, scores, depth, evaluation): + """Zero entities first reached at masked distances; upstream masks heads also in training.""" + masked = [d for d in self.masked_distances if evaluation or d == 0] + if masked and depth is not None: + scores = scores.masked_fill(torch.isin(depth, torch.tensor(masked, device=depth.device)), 0) + return scores + + def _all_scores(self, heads, relations, edges, fused): + values, ids = self._prompts(relations) + if fused: + scores, depth = fused_scores(self, heads, relations, ids, values, edges) + else: + prompts = torch.stack([values[q] for q in ids]) + scores, depth = self._expand_scores(heads, relations, prompts, edges) + return self._mask_distances(scores, depth, not self.training) + + def _score(self, heads, relations, candidates, query_relations, edges): + # Query conditioning uses the actual (possibly inverse) relation, as upstream does. + self._require_graph() + output = [] + inference = inference_only(self) + fused = inference and fused_reasoning_supported(self, edges) + # Batched GPU reasoning is row-wise; the PyTorch path scores each query + # alone during inference, so a row never depends on its batch. + chunk = self.query_batch_size if fused or not inference else 1 + ids = host_ids(relations) + for start in range(0, len(heads), chunk): + sl = slice(start, start + chunk) + part = relations[sl] + part._dicee_ids = ids[sl] + scores = self._all_scores(heads[sl], part, edges, fused) + output.append(candidate_features(scores[..., None], candidate_slice(candidates, sl)).squeeze(-1)) + return torch.cat(output) if output else next(self.parameters()).new_empty((0, candidates.shape[1])) diff --git a/dicee/models/kgicl_prompts.py b/dicee/models/kgicl_prompts.py new file mode 100644 index 000000000..669ee373a --- /dev/null +++ b/dicee/models/kgicl_prompts.py @@ -0,0 +1,233 @@ +"""KG-ICL prompt graphs: deterministic example sampling and extraction. + +Semantics follow nju-websoft/KG-ICL at UPSTREAM_COMMIT (``utils_all.py`` and +``data_loader.py``), with the prompt construction of the released datasets: +an example fact ``(u, q, v)``, every entity ``x`` within ``hops`` of both ``u`` +and ``v`` with ``dist(x,u) + dist(x,v) <= hops``, up to ``open_nodes`` further +entities within ``hops`` of both, and every fact among the selected entities. +Upstream draws examples and the additional entities with unseeded NumPy +calls. Here every draw comes from a generator seeded by the prompt seed and +the public ID of the direct relation, so a graph and relation always yield +the same prompt graphs, independently of query batches and call order. +docs/kgicl.md lists the upstream defects that are corrected here. +""" +import hashlib +from dataclasses import dataclass +from typing import Optional + +import torch + +UNREACHED = 1 << 30 + + +def prompt_generator(*values: int) -> torch.Generator: + """A CPU generator seeded from integer values, stable across processes and platforms.""" + text = 'kgicl-prompt:' + ':'.join(str(int(v)) for v in values) + return torch.Generator().manual_seed(int(hashlib.sha256(text.encode()).hexdigest()[:15], 16)) + + +@dataclass(frozen=True, eq=False) +class PromptGraph: + """One prompt graph in encoder layout, specialized for a query relation. + + ``edge_index`` holds local (head, tail) pairs of every prompt fact and its + inverse. ``edge_type`` uses internal relation IDs: direct relations in + ``[0, R)`` and their inverses in ``[R, 2R)``. ``labels`` are the distance + tokens ``(dist to u, dist to v)`` of the example fact ``(u, q, v)``; + ``head`` and ``tail`` are the local nodes that receive the head and tail + tokens. For an inverse query relation they are swapped, as upstream does. + """ + + edge_index: torch.Tensor + edge_type: torch.Tensor + labels: torch.Tensor + head: int + tail: int + + @property + def num_nodes(self) -> int: + return len(self.labels) + + +@dataclass(frozen=True, eq=False) +class PromptExample: + """An example fact's extracted prompt graph, before specialization to a query relation. + + ``entities`` maps local nodes to graph entities: node 0 is the example head + and, unless the example is a self-loop, node 1 its tail. ``triples`` are the + deduplicated direct facts among these entities in local IDs. + """ + entities: torch.Tensor # [n] + triples: torch.Tensor # [E, 3] + labels: torch.Tensor # [n, 2] + self_loop: bool + + +def _csr(keys, values, size): + order = keys.argsort(stable=True) + counts = torch.bincount(keys, minlength=size) + return torch.cat((counts.new_zeros(1), counts.cumsum(0))), values[order] + + +def _gather(offsets, values, nodes): + """Concatenated CSR rows of ``nodes``, in node order.""" + starts, counts = offsets[nodes], offsets[nodes + 1] - offsets[nodes] + total = int(counts.sum()) + if not total: + return values[:0] + rows = torch.arange(len(nodes)).repeat_interleave(counts) + index = starts[rows] + torch.arange(total) - (counts.cumsum(0) - counts)[rows] + return values[index] + + +class PromptSampler: + """Deterministic prompt graphs of an attached graph, kept on the host. + + Args: + triples: ``[N, 3]`` internal (head, relation, tail) facts including + inverse facts, as attached by ``GraphKGE.set_graph``. + num_entities: Entity vocabulary size. + num_direct: Number of direct relations ``R``; inverse IDs are ``r + R``. + public_relations: Public ID of each internal direct relation, used for seeding. + shots: Prompt graphs per query relation (upstream ``shot``). + hops: Distance bound ``k`` (upstream ``hop``/``path_hop``). + open_nodes: Additional entities beyond ``dist(x,u) + dist(x,v) <= k`` + (upstream ``enclosing=False``: 50). + seed: Prompt seed. + """ + + def __init__(self, triples: torch.Tensor, num_entities: int, num_direct: int, public_relations: list, + shots: int = 5, hops: int = 3, open_nodes: int = 50, seed: int = 0): + triples = triples.detach().cpu().long() + direct = triples[triples[:, 1] < num_direct].unique(dim=0) + self.num_entities, self.num_direct = num_entities, num_direct + self.public_relations = list(public_relations) + self.shots, self.hops, self.open_nodes, self.seed = shots, hops, open_nodes, seed + # Facts of each relation in canonical (head, tail) order. + relation_order = (direct[:, 1] * num_entities + direct[:, 0]) * num_entities + direct[:, 2] + direct = direct[relation_order.argsort()] + self.relation_offsets = torch.cat((torch.zeros(1, dtype=torch.long), + torch.bincount(direct[:, 1], minlength=num_direct).cumsum(0))) + self.relation_facts = direct[:, [0, 2]] + # Facts by head, for induced subgraphs, and an undirected simple adjacency for distances. + self.head_offsets, self.head_facts = _csr(direct[:, 0], direct, num_entities) + pairs = torch.cat((direct[:, [0, 2]], direct[:, [2, 0]])) + pairs = pairs[pairs[:, 0] != pairs[:, 1]].unique(dim=0) + self.adjacency_offsets, self.adjacency = _csr(pairs[:, 0], pairs[:, 1], num_entities) + self._examples: dict[int, list[PromptExample]] = {} + + def distances(self, source: int) -> torch.Tensor: + """Undirected hop distances from ``source`` up to ``hops``; farther entities are ``UNREACHED``.""" + distance = torch.full((self.num_entities,), UNREACHED, dtype=torch.long) + distance[source] = 0 + frontier = torch.tensor([source]) + for step in range(1, self.hops + 1): + reached = torch.zeros(self.num_entities, dtype=torch.bool) + reached[_gather(self.adjacency_offsets, self.adjacency, frontier)] = True + frontier = (reached & (distance == UNREACHED)).nonzero().flatten() + if not len(frontier): + break + distance[frontier] = step + return distance + + def extract(self, head: int, tail: int, generator: Optional[torch.Generator] = None) -> PromptExample: + """Prompt graph of the example fact ``(head, ?, tail)``, before query specialization.""" + to_head, to_tail = self.distances(head), self.distances(tail) + within = (to_head <= self.hops) & (to_tail <= self.hops) + within[[head, tail]] = False + candidates = within.nonzero().flatten() + total = to_head[candidates] + to_tail[candidates] + close, far = candidates[total <= self.hops], candidates[total > self.hops] + if len(far) > self.open_nodes: + far = far[torch.randperm(len(far), generator=generator)[:self.open_nodes]].sort().values + rest = torch.cat((close, far)).sort().values + nodes = torch.cat((torch.tensor([head] if head == tail else [head, tail]), rest)) + local = torch.full((self.num_entities,), -1, dtype=torch.long) + local[nodes] = torch.arange(len(nodes)) + facts = _gather(self.head_offsets, self.head_facts, nodes) + facts = facts[local[facts[:, 2]] >= 0] + triples = torch.stack((local[facts[:, 0]], facts[:, 1], local[facts[:, 2]]), 1) + labels = torch.stack((to_head[nodes], to_tail[nodes]), 1) + if head == tail: + labels[0] = torch.tensor([0, 0]) + else: + labels[0], labels[1] = torch.tensor([0, 1]), torch.tensor([1, 0]) + return PromptExample(nodes, triples, labels, head == tail) + + def examples(self, relation: int) -> list: + """Up to ``shots`` extracted example facts of a direct relation, in sampling order.""" + if relation not in self._examples: + public = self.public_relations[relation] + start, end = int(self.relation_offsets[relation]), int(self.relation_offsets[relation + 1]) + facts = self.relation_facts[start:end] + order = torch.randperm(len(facts), generator=prompt_generator(self.seed, public))[:self.shots] + self._examples[relation] = [self.extract(int(facts[i, 0]), int(facts[i, 1]), + prompt_generator(self.seed, public, slot)) + for slot, i in enumerate(order.tolist())] + return self._examples[relation] + + def prompts(self, relation: int, slots: Optional[list] = None) -> list: + """Prompt graphs of an internal query relation, one per shot. + + Slot ``j`` uses example ``j mod n`` when only ``n < shots`` examples exist. + ``slots`` overrides that choice with explicit example indices (training draws). + """ + direct = relation % self.num_direct + examples = self.examples(direct) + if not examples: + # Upstream's fallback for a relation without facts: two nodes joined by + # one query-relation edge, without its inverse. + head, tail = (0, 1) if relation < self.num_direct else (1, 0) + dummy = PromptGraph(torch.tensor([[0], [1]]), torch.tensor([relation]), + torch.tensor([[0, 1], [1, 0]]), head, tail) + return [dummy] * self.shots + if slots is None: + slots = [j % len(examples) for j in range(self.shots)] + return [specialize(examples[j], relation, self.num_direct) for j in slots] + + +def specialize(example: PromptExample, relation: int, num_direct: int) -> PromptGraph: + """Add inverse facts and place the head/tail tokens for a direct or inverse query relation.""" + triples = example.triples + inverse = torch.stack((triples[:, 2], triples[:, 1] + num_direct, triples[:, 0]), 1) + both = torch.cat((triples, inverse)) + if example.self_loop: + head = tail = 0 + else: + head, tail = (0, 1) if relation < num_direct else (1, 0) + return PromptGraph(both[:, [0, 2]].T.contiguous(), both[:, 1].contiguous(), example.labels, head, tail) + + +def answer_distance_rates(facts: torch.Tensor, background: torch.Tensor, num_entities: int, + max_distance: int = 9) -> list: + """Fractions of reachable ``facts`` whose endpoints lie at each undirected distance in ``background``. + + Upstream (``utils.get_distance``) computes these rates from validation facts + on the validation background graph and suppresses entities first reached at + a distance whose rate is below 0.02. Unreachable facts are not counted. + """ + facts, background = torch.as_tensor(facts).long().cpu(), torch.as_tensor(background).long().cpu() + pairs = torch.cat((background[:, [0, 2]], background[:, [2, 0]])) + pairs = pairs[pairs[:, 0] != pairs[:, 1]].unique(dim=0) + offsets, adjacency = _csr(pairs[:, 0], pairs[:, 1], num_entities) + counts = [0] * (max_distance + 1) + reachable = 0 + for source in facts[:, 0].unique().tolist(): + targets = facts[facts[:, 0] == source, 2] + distance = torch.full((num_entities,), UNREACHED, dtype=torch.long) + distance[source] = 0 + frontier = torch.tensor([source]) + remaining = targets.unique() + step = 0 + while len(frontier) and (distance[remaining] == UNREACHED).any(): + step += 1 + neighbors = _gather(offsets, adjacency, frontier).unique() + frontier = neighbors[distance[neighbors] == UNREACHED] + distance[frontier] = step + found = distance[targets] + found = found[found != UNREACHED] + reachable += len(found) + for value in found.tolist(): + if value <= max_distance: + counts[value] += 1 + return [count / reachable if reachable else float('nan') for count in counts] diff --git a/dicee/models/trix.py b/dicee/models/trix.py index d44041f88..709d61dd5 100644 --- a/dicee/models/trix.py +++ b/dicee/models/trix.py @@ -11,7 +11,7 @@ import torch from torch import nn -from ._inference import candidate_features, candidate_slice, conditioned_linear, conditioned_score, inference_only +from ._inference import candidate_features, candidate_slice, conditioned_linear, conditioned_score, host_ids, inference_only, to_device from .graph_model import GraphKGE, RelationGraphKGE from .ultra import RelationalConv @@ -141,7 +141,9 @@ def clear_inference_cache(self): def _first_relation_step(self, query_relations): weight = next(self.parameters()) boundary = weight.new_zeros(len(query_relations), 2 * self.num_direct_relations, self.dim) - boundary[torch.arange(len(query_relations), device=query_relations.device), query_relations] = 1 + # A device scalar avoids a host copy (and stream synchronization) per call. + boundary.index_put_((torch.arange(len(query_relations), device=query_relations.device), query_relations), + boundary.new_ones(())) # All entities initially have the same label. Project a single label # per query and broadcast it *after* the MLP, retaining the graph layout. entities = weight.new_ones(len(query_relations), 1, self.dim) @@ -160,12 +162,12 @@ def _cached_initial_relations(self, query_relations): if token != self._initial_cache_token: self._initial_cache.clear() self._initial_cache_token = token - ids = query_relations.tolist() + ids = host_ids(query_relations) missing = list(dict.fromkeys(q for q in ids if q not in self._initial_cache)) values = {q: self._initial_cache[q] for q in set(ids) if q in self._initial_cache} for start in range(0, len(missing), self.query_batch_size): keys = missing[start:start + self.query_batch_size] - hidden = self._first_relation_step(query_relations.new_tensor(keys)) + hidden = self._first_relation_step(to_device(torch.tensor(keys), query_relations.device)) values.update((key, value.clone()) for key, value in zip(keys, hidden)) for key in dict.fromkeys(ids): self._initial_cache[key] = values[key] @@ -182,7 +184,7 @@ def _reason(self, heads, relations, query_relations, edges, initial=None): features, query = self.entity_model_1.features(edges, num_entities, hidden, heads, relations, split=True) entities = conditioned_linear(self.relation_model.node_mlp, features, query) boundary = weight.new_zeros(len(heads), 2 * self.num_direct_relations, self.dim) - boundary[torch.arange(len(heads), device=heads.device), query_relations] = 1 + boundary.index_put_((torch.arange(len(heads), device=heads.device), query_relations), boundary.new_ones(())) for i in (1, 2): hidden = self.relation_model.step(i, hidden, boundary, self.relation_graph, entities) return hidden diff --git a/dicee/models/ultra.py b/dicee/models/ultra.py index a80685107..5cdb0189a 100644 --- a/dicee/models/ultra.py +++ b/dicee/models/ultra.py @@ -35,7 +35,7 @@ from torch import nn from ._fused_message import fused_distmult_sum, fused_norm_relu -from ._inference import candidate_features, candidate_slice, compiled_conv_update, conditioned_score, inference_only +from ._inference import cached_relation_batch, candidate_features, candidate_slice, compiled_conv_update, conditioned_score, host_ids, inference_only, to_device from .graph_model import GraphKGE UPSTREAM_COMMIT = "427966ad8ed60420eef034063d44f3153addff90" @@ -170,6 +170,11 @@ def __init__(self, args): self.set_inference_backend(args.get('graph_inference_backend', 'auto')) for layer in (*self.relation_model.layers, *self.entity_model.layers): layer.inference_compile = args.get('graph_inference_compile', False) + # 'query' conditions an inverse tail query on the inverse relation, as UltraQuery's projection does, for weights + # trained that way; the default 'direct' is ULTRA link prediction. + self.relation_conditioning = args.get('ultra_relation_conditioning', 'direct') + if self.relation_conditioning not in ('direct', 'query'): + raise ValueError('ultra_relation_conditioning must be direct or query') def clear_inference_cache(self): self._relation_cache = OrderedDict() @@ -198,14 +203,14 @@ def _cached_relations(self, query_relations): if token != self._relation_cache_token: self.clear_inference_cache() self._relation_cache_token = token - ids = query_relations.tolist() + ids = host_ids(query_relations) missing = list(dict.fromkeys(q for q in ids if q not in self._relation_cache)) # Keep current-call values alive even when they exceed the LRU capacity. values = {q: self._relation_cache[q] for q in set(ids) if q in self._relation_cache} for start in range(0, len(missing), self.query_batch_size): keys = missing[start:start + self.query_batch_size] rels = self.relation_model(self.rel_edge_index, self.rel_edge_type, 2 * self.num_direct_relations, - query_relations.new_tensor(keys)) + to_device(torch.tensor(keys), query_relations.device)) for key, value in zip(keys, rels): values[key] = value.clone() for key in dict.fromkeys(ids): @@ -228,7 +233,7 @@ def _cached_projections(self, rels, ids, token): missing = list(dict.fromkeys(q for q in ids if q not in self._projection_cache)) values = {q: self._projection_cache[q] for q in set(ids) if q in self._projection_cache} if missing: - indices = rels.new_tensor([ids.index(q) for q in missing], dtype=torch.long) + indices = to_device(torch.tensor([ids.index(q) for q in missing], dtype=torch.long), rels.device) source = rels.index_select(0, indices) projected = [layer.relation_projection(source) for layer in self.entity_model.layers] for i, key in enumerate(missing): @@ -238,7 +243,7 @@ def _cached_projections(self, rels, ids, token): self._projection_cache.move_to_end(key) while len(self._projection_cache) > capacity: self._projection_cache.popitem(last=False) - return tuple(torch.stack([values[q][i] for q in ids]) for i in range(self.num_layers)) + return tuple(cached_relation_batch({q: value[i] for q, value in values.items()}, ids) for i in range(self.num_layers)) def _score(self, heads, relations, candidates, query_relations, edges): output = [] diff --git a/dicee/query_answering/__init__.py b/dicee/query_answering/__init__.py new file mode 100644 index 000000000..b02b671e0 --- /dev/null +++ b/dicee/query_answering/__init__.py @@ -0,0 +1,14 @@ +"""Shared complex-query inference and trainable score adapters.""" + +from .adapter_training import AdapterFitResult, AdapterQuery, AdapterTrainingData, fit_query_adapter, prepare_adapter_data +from .benchmark import benchmark_model, evaluate_benchmark, summarize_benchmarks +from .context import QueryContext +from .datasets import BENCHMARK_DATASETS, PLUS_H_DATASETS, BenchmarkQuery, QueryBenchmark, load_benchmark +from .engine import QueryAnswerer +from .method_evaluation import evaluate_method +from .score_adapter import QueryScoreAdapter + +__all__ = ['QueryAnswerer', 'QueryContext', 'QueryScoreAdapter', 'evaluate_method', 'AdapterQuery', 'AdapterTrainingData', + 'AdapterFitResult', 'prepare_adapter_data', 'fit_query_adapter', + 'BENCHMARK_DATASETS', 'PLUS_H_DATASETS', 'BenchmarkQuery', 'QueryBenchmark', 'load_benchmark', + 'benchmark_model', 'evaluate_benchmark', 'summarize_benchmarks'] diff --git a/dicee/query_answering/__main__.py b/dicee/query_answering/__main__.py new file mode 100644 index 000000000..448fba832 --- /dev/null +++ b/dicee/query_answering/__main__.py @@ -0,0 +1,103 @@ +"""Fit query adapters and run benchmarks: python -m dicee.query_answering --help.""" + +import argparse +import json +from pathlib import Path + +from .adapter_training import TRAINING_SHAPES, AdapterTrainingData, fit_query_adapter, prepare_adapter_data +from .benchmark import add_benchmark_parser, run_benchmark_cli +from .context import QueryContext +from .score_adapter import FEATURE_COUNTS, QueryScoreAdapter, canonical_feature_mode + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + add_benchmark_parser(commands) + prepare = commands.add_parser('prepare', help='Mask source facts and generate disjoint logical queries') + prepare.add_argument('--source', type=Path, required=True, help='QueryContext JSON: triples, num_entities, num_relations, inverse_relations') + prepare.add_argument('--name', default='source') + prepare.add_argument('--output', type=Path, required=True) + prepare.add_argument('--mask-fraction', type=float, default=.3) + prepare.add_argument('--train-per-shape', type=int, default=96) + prepare.add_argument('--validation-per-shape', type=int, default=32) + prepare.add_argument('--seed', type=int, default=2026090851) + prepare.add_argument('--shapes', nargs='+', choices=TRAINING_SHAPES, default=['2i', '3i']) + fit = commands.add_parser('fit', help='Fit adapters for a frozen DICE experiment') + fit.add_argument('--experiment', required=True) + fit.add_argument('--data', type=Path, nargs='+', required=True) + fit.add_argument('--output', type=Path, required=True, help='Output directory for adapter, report, and score banks') + fit.add_argument('--feature-mode', type=canonical_feature_mode, choices=FEATURE_COUNTS, + default='context_scores', help='Adapter features (default: context_scores, linear head)') + fit.add_argument('--epochs', type=int, default=20) + fit.add_argument('--batch-size', type=int, default=8) + fit.add_argument('--row-batch-size', type=int, default=8) + fit.add_argument('--learning-rate', type=float, default=.02) + fit.add_argument('--observed-mix', type=float, default=1.) + fit.add_argument('--bias-bound', type=float, default=4.) + fit.add_argument('--scale-bound', type=lambda v: None if v == 'none' else float(v), default=2.) + fit.add_argument('--beam-size', type=int, default=64) + fit.add_argument('--tnorm', choices=['prod', 'min'], default='prod') + fit.add_argument('--normalization', choices=['none', 'standard'], default='none') + fit.add_argument('--hidden-dim', type=int, default=0) + fit.add_argument('--validation-every', type=int) + fit.add_argument('--early-stopping-patience', type=int, help='Epochs without source-validation MRR improvement') + fit.add_argument('--seed', type=int, default=2026090851) + fit.add_argument('--samples', type=int) + fit.add_argument('--device', default='cpu') + fit.add_argument('--training-device', choices=['cpu', 'cuda'], default='cpu') + fit.add_argument('--validation-device', choices=['cpu', 'cuda']) + fit.add_argument('--cpu-threads', type=int) + fit.add_argument('--training-cache-mb', type=int, default=1536) + fit.add_argument('--device-cache-mb', type=int, default=512) + fit.add_argument('--compare-global', action='store_true', help='Also fit a two-parameter baseline, reusing frozen score banks') + args = parser.parse_args() + if args.command == 'benchmark': + run_benchmark_cli(args) + return + if args.command == 'prepare': + data = prepare_adapter_data(QueryContext(**json.loads(args.source.read_text())), name=args.name, + mask_fraction=args.mask_fraction, train_per_shape=args.train_per_shape, + validation_per_shape=args.validation_per_shape, seed=args.seed, shapes=args.shapes) + args.output.parent.mkdir(parents=True, exist_ok=True) + data.save(args.output) + print(f'Saved {len(data.train)} training and {len(data.validation)} validation queries to {args.output}') + return + from ..knowledge_graph_embeddings import KGE + if args.cpu_threads is not None: + if args.cpu_threads < 1: + parser.error('--cpu-threads must be positive') + import torch + torch.set_num_threads(args.cpu_threads) + kge = KGE(path=args.experiment) + kge.to(args.device) + sources = [AdapterTrainingData.load(path) for path in args.data] + args.output.mkdir(parents=True, exist_ok=True) + modes = [args.feature_mode] + if args.compare_global and args.feature_mode != 'global': + modes.append('global') + reports = {} + for mode in modes: + result = fit_query_adapter(kge.model, sources, feature_mode=mode, observed_mix=args.observed_mix, + epochs=args.epochs, batch_size=args.batch_size, learning_rate=args.learning_rate, + row_batch_size=args.row_batch_size, seed=args.seed, samples=args.samples, + cache_dir=args.output / 'score-banks', bias_bound=args.bias_bound, + scale_bound=args.scale_bound, beam_size=args.beam_size, tnorm=args.tnorm, + normalization=args.normalization, hidden_dim=args.hidden_dim, + validation_every=args.validation_every, + early_stopping_patience=args.early_stopping_patience, + training_device=args.training_device, training_cache_bytes=args.training_cache_mb * 2**20, + validation_device=args.validation_device, + device_cache_bytes=args.device_cache_mb * 2**20, + checkpoint_path=args.output / f'{mode or "default"}-fit-state.pt') + name = result.adapter.feature_mode + path = args.output / f'{name}.json' + result.adapter.save(path) + QueryScoreAdapter.load(path, model=kge.model) + reports[name] = dict(history=result.history, validation=result.validation, metadata=result.adapter.metadata) + print(f'Saved {name} adapter to {path}') + (args.output / 'report.json').write_text(json.dumps(reports, indent=2, allow_nan=False) + '\n') + + +if __name__ == '__main__': + main() diff --git a/dicee/query_answering/_adapter_cache.py b/dicee/query_answering/_adapter_cache.py new file mode 100644 index 000000000..38dc20fa4 --- /dev/null +++ b/dicee/query_answering/_adapter_cache.py @@ -0,0 +1,225 @@ +"""Bounded host/device caches for frozen scores and parameter-independent features.""" + +import hashlib +import json +import sqlite3 +import time +from collections import OrderedDict +from pathlib import Path + +import numpy as np +import torch + +from ._checkpoint import scoring_fingerprint +from .context import attached_context, fingerprint, state_fingerprint +from .engine import AtomicScorer + + +class TrainingRows: + def __init__(self, model, context, *, cache_dir, row_batch_size, seed, samples, + cache_bytes=128 * 1024**2, disk_bytes=8 * 1024**3, + device='cpu', device_cache_bytes=0): + from ..models._inference import float32_precision_token + + self.model, self.context = model, context + self.device = torch.device(device) + if self.device.type == 'cuda' and self.device.index is None: + self.device = torch.device('cuda', torch.cuda.current_device()) + if min(cache_bytes, disk_bytes, device_cache_bytes) < 0: + raise ValueError('Cache budgets must be nonnegative') + self.row_batch_size, self.seed, self.samples = row_batch_size, seed, samples + self.cache_bytes, self.cache_used, self.cache = cache_bytes, 0, OrderedDict() + self.device_cache_bytes, self.device_cache_used, self.device_cache = device_cache_bytes, 0, OrderedDict() + self.device_rows, self.next_block = {}, 0 + self.feature_cache = OrderedDict() + self.stats = dict(host_hits=0, device_hits=0, disk_rows=0, scored_rows=0, feature_preparations=0, seconds=0.) + self.disk_limit = disk_bytes if cache_dir is not None else min(disk_bytes, cache_bytes) + from ..models.flock import Flock + # Only sampled backbones (Flock walks) depend on the scoring seed; others share rows across seeds. + sampled = isinstance(model, Flock) + settings = dict(version=2, backbone=state_fingerprint(model), context=context.identity, + scoring_code=scoring_fingerprint(), architecture=str(model), + args={k: str(v) for k, v in getattr(model, 'args', {}).items()}, + row_batch_size=row_batch_size, seed=seed if sampled else None, samples=samples if sampled else None, + precision=float32_precision_token(), torch=str(torch.__version__), + deterministic=torch.are_deterministic_algorithms_enabled(), + autocast=torch.is_autocast_enabled(next(model.parameters()).device.type), + device=str(next(model.parameters()).device)) + if hasattr(model, 'prompt_identity'): + # KG-ICL rows depend on its prompt sampling; other backbones keep their identities. + settings['prompts'] = model.prompt_identity() + self.identity = fingerprint(settings) + path = ':memory:' + if cache_dir is not None: + Path(cache_dir).mkdir(parents=True, exist_ok=True) + path = str(Path(cache_dir) / f'{self.identity}.sqlite3') + self.connection = sqlite3.connect(path) + self.connection.execute('PRAGMA cache_size=-8192') + self.connection.execute('CREATE TABLE IF NOT EXISTS metadata (identity TEXT PRIMARY KEY, settings TEXT)') + existing = self.connection.execute('SELECT identity, settings FROM metadata').fetchall() + expected = (self.identity, json.dumps(settings, sort_keys=True)) + if existing and existing != [expected]: + raise ValueError('Training row cache identity mismatch') + self.connection.execute('INSERT OR IGNORE INTO metadata VALUES (?, ?)', expected) + self.connection.execute('CREATE TABLE IF NOT EXISTS rows (head INTEGER, relation INTEGER, dtype TEXT, value BLOB, sha256 TEXT, PRIMARY KEY(head, relation))') + self.connection.execute('CREATE TABLE IF NOT EXISTS features (mode TEXT, head INTEGER, relation INTEGER, value BLOB, sha256 TEXT, PRIMARY KEY(mode, head, relation))') + self.feature_code = fingerprint({name: hashlib.sha256(Path(__file__).with_name(name).read_bytes()).hexdigest() + for name in ('score_adapter.py', '_adapter_cache.py')}) + self.connection.commit() + self.disk_used = self.connection.execute('SELECT COALESCE(SUM(LENGTH(value)), 0) FROM rows').fetchone()[0] + + def prepared(self, adapter, conditions, *, device=None): + start = time.perf_counter() + target = self.device if device is None else torch.device(device) + if target.type == 'cuda' and target.index is None: + target = torch.device('cuda', torch.cuda.current_device()) + if target.type != 'cpu' and target != self.device: + raise ValueError('Prepared rows require CPU or the configured cache device') + mode = json.dumps((self.feature_code, adapter.feature_mode, adapter.normalization)) + ready, pending = {}, [] + for pair in dict.fromkeys(conditions): + key = (mode, pair) + if target.type != 'cpu' and key in self.device_rows: + block, index = self.device_rows[key] + ready[pair] = tuple(v[index] for v in self.device_cache[block][1]) + self.device_cache.move_to_end(block) + self.stats['device_hits'] += 1 + elif key in self.cache: + ready[pair] = self.cache[key] + self.cache.move_to_end(key) + self.stats['host_hits'] += 1 + else: + pending.append(pair) + records, features = {}, {} + for offset in range(0, len(pending), 256): + batch = pending[offset:offset + 256] + where = ','.join('(?,?)' for _ in batch) + args = [v for pair in batch for v in pair] + records.update({(h, r): (dtype, value, digest) for h, r, dtype, value, digest in self.connection.execute( + f'WITH wanted(head,relation) AS (VALUES {where}) ' + 'SELECT r.head,r.relation,r.dtype,r.value,r.sha256 FROM wanted w ' + 'JOIN rows r ON r.head=w.head AND r.relation=w.relation', args)}) + features.update({(h, r): (value, digest) for h, r, value, digest in self.connection.execute( + f'WITH wanted(head,relation) AS (VALUES {where}) ' + 'SELECT f.head,f.relation,f.value,f.sha256 FROM wanted w ' + 'JOIN features f ON f.head=w.head AND f.relation=w.relation WHERE f.mode=?', [*args, mode])}) + missing = [] + for pair in pending: + if pair in records: + dtype, value, digest = records[pair] + if hashlib.sha256(value).hexdigest() != digest: + raise ValueError('Corrupt training score row') + raw = torch.from_numpy(np.frombuffer(value, dtype=dtype).copy()) + self.stats['disk_rows'] += 1 + self._prepare(adapter, pair, raw, mode, ready, features.get(pair)) + else: + missing.append(pair) + if missing: + # Leave transient backbone activations room on a shared GPU. + self.device_cache.clear() + self.device_rows.clear() + self.device_cache_used = 0 + with attached_context(self.model, self.context): + scorer = AtomicScorer(self.model, self.context, row_batch_size=self.row_batch_size, + seed=self.seed, samples=self.samples) + raw = scorer.rows(missing).cpu() + self.stats['scored_rows'] += len(missing) + for pair, row in zip(missing, raw): + array = row.numpy() + value = array.tobytes() + self.connection.execute('INSERT INTO rows VALUES (?, ?, ?, ?, ?)', + (*pair, array.dtype.str, value, hashlib.sha256(value).hexdigest())) + self.disk_used += len(value) + self._prepare(adapter, pair, row, mode, ready) + while self.disk_used > self.disk_limit: + oldest = self.connection.execute('SELECT rowid, LENGTH(value) FROM rows ORDER BY rowid LIMIT 128').fetchall() + self.connection.executemany('DELETE FROM rows WHERE rowid=?', [(row,) for row, _ in oldest]) + self.disk_used -= sum(size for _, size in oldest) + if self.connection.in_transaction: + self.connection.commit() + if target.type != 'cpu': + pairs = [pair for pair, values in ready.items() if values[0].device != target] + if pairs: + block = tuple(torch.stack([ready[pair][i] for pair in pairs]).to(target) for i in range(3)) + self._remember_device([(mode, pair) for pair in pairs], block) + for index, pair in enumerate(pairs): + ready[pair] = tuple(v[index] for v in block) + self.stats['seconds'] += time.perf_counter() - start + values = tuple(torch.stack([ready[pair][i] for pair in conditions]) for i in range(3)) + return values[0].to(torch.float64), values[1], values[2] + + def _prepare(self, adapter, pair, raw, mode, ready, saved=None): + if raw.shape != (self.context.num_entities,) or not torch.isfinite(raw).all(): + raise ValueError('Invalid training score row') + stored = raw.clone() if raw._base is not None else raw + raw = raw.to(torch.float64) + key = (mode, pair) + fixed = self.feature_cache.get(key) + if fixed is None and saved is not None: + value, digest = saved + if hashlib.sha256(value).hexdigest() != digest: + raise ValueError('Corrupt training features') + fixed = torch.from_numpy(np.frombuffer(value, dtype=np.float64).copy()) + if fixed is None: + observed, base = self.context.features([pair], device='cpu') + _, _, features = adapter.prepare(raw[None], observed, base) + mean, std = raw.new_zeros(1), raw.new_ones(1) + if adapter.normalization == 'standard': + mean = raw[None].mean(1) + std = raw[None].std(1, correction=0).clamp_min(1e-6) + fixed = torch.cat((features[0], mean, std)).detach() + value = fixed.numpy().tobytes() + self.connection.execute('INSERT OR REPLACE INTO features VALUES (?, ?, ?, ?, ?)', + (mode, *pair, value, hashlib.sha256(value).hexdigest())) + self.stats['feature_preparations'] += 1 + self.feature_cache[key] = fixed + self.feature_cache.move_to_end(key) + while len(self.feature_cache) > 4096: + self.feature_cache.popitem(last=False) + observed = torch.zeros(self.context.num_entities, dtype=torch.bool) + observed[sorted(self.context.outgoing.get(pair[0], {}).get(pair[1], set()))] = True + if adapter.normalization == 'standard': + raw = (raw - fixed[-2]) / fixed[-1] + else: + raw = stored + values = raw, observed, fixed[:-2] + ready[pair] = values + self._remember('', key, values) + + def _remember(self, prefix, key, values): + cache = getattr(self, prefix + 'cache') + used, limit = getattr(self, prefix + 'cache_used'), getattr(self, prefix + 'cache_bytes') + size = sum(v.numel() * v.element_size() for v in values) + if size <= limit: + old = cache.pop(key, ()) + used -= sum(v.numel() * v.element_size() for v in old) + while cache and used + size > limit: + _, old = cache.popitem(last=False) + used -= sum(v.numel() * v.element_size() for v in old) + cache[key] = values + setattr(self, prefix + 'cache_used', used + size) + + def _remember_device(self, keys, values): + # Charge whole allocations: evicted row views must not hide live blocks. + size = sum(v.numel() * v.element_size() for v in values) + if size > self.device_cache_bytes: + return + while self.device_cache and self.device_cache_used + size > self.device_cache_bytes: + block, (old_keys, old) = self.device_cache.popitem(last=False) + self.device_cache_used -= sum(v.numel() * v.element_size() for v in old) + for key in old_keys: + if self.device_rows.get(key, (None,))[0] == block: + del self.device_rows[key] + block = self.next_block + self.next_block += 1 + self.device_cache[block] = (keys, values) + self.device_cache_used += size + for index, key in enumerate(keys): + self.device_rows[key] = block, index + + def close(self): + self.connection.close() + self.cache.clear() + self.device_cache.clear() + self.device_rows.clear() + self.feature_cache.clear() diff --git a/dicee/query_answering/_checkpoint.py b/dicee/query_answering/_checkpoint.py new file mode 100644 index 000000000..9e7b6e855 --- /dev/null +++ b/dicee/query_answering/_checkpoint.py @@ -0,0 +1,112 @@ +"""Atomic, single-writer benchmark checkpoints.""" + +import hashlib +import json +import os +from pathlib import Path + +from .context import fingerprint + + +def write_json(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + '.tmp') + with temporary.open('w') as stream: + json.dump(value, stream, indent=2, allow_nan=False) + stream.write('\n') + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + + +class BenchmarkCheckpoint: + def __init__(self, directory, identity): + self.directory = Path(directory) + self.identity = fingerprint(identity) + self.lock = None + + def __enter__(self): + self.directory.mkdir(parents=True, exist_ok=True) + self.lock = (self.directory / '.lock').open('a+b') + try: + if os.name == 'nt': + import msvcrt + self.lock.write(b'0') + self.lock.flush() + self.lock.seek(0) + msvcrt.locking(self.lock.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(self.lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as error: + self.lock.close() + raise RuntimeError(f'Benchmark already running: {self.directory}') from error + try: + manifest = self.directory / 'manifest.json' + expected = dict(version=1, identity=self.identity) + if manifest.exists(): + if json.loads(manifest.read_text()) != expected: + raise ValueError('Benchmark inputs/settings changed; use a new checkpoint directory') + else: + write_json(manifest, expected) + except BaseException: + self.__exit__(None, None, None) + raise + return self + + def load(self): + path = self.directory / 'state.json' + if not path.exists(): + return None + record = json.loads(path.read_text()) + if record.get('identity') != self.identity or record.get('sha256') != fingerprint(record['state']): + raise ValueError('Corrupt benchmark checkpoint') + return record['state'] + + def save(self, state): + write_json(self.directory / 'state.json', dict(identity=self.identity, sha256=fingerprint(state), state=state)) + + def __exit__(self, *_): + if self.lock is not None: + if os.name == 'nt': + import msvcrt + self.lock.seek(0) + msvcrt.locking(self.lock.fileno(), msvcrt.LK_UNLCK, 1) + self.lock.close() + + +def checksum(path): + with Path(path).open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def source_fingerprint(paths, root): + """Identity of source files, keyed by their path relative to ``root``.""" + return fingerprint({str(Path(path).relative_to(root)): checksum(path) for path in paths}) + + +def implementation_fingerprint(package=None): + """Code that scores and evaluates queries: query answering, models and filtering.""" + package = Path(package) if package is not None else Path(__file__).parents[1] + paths = [*(package / 'query_answering').rglob('*.py'), *(package / 'models').glob('*.py'), + package / 'evaluation' / '_filtering.py'] + return source_fingerprint(paths, package) + + +def scoring_fingerprint(package=None): + """Code that produces atomic query rows: models, engine and context.""" + package = Path(package) if package is not None else Path(__file__).parents[1] + paths = [*(package / 'models').glob('*.py'), *(package / 'query_answering' / name for name in ('context.py', 'engine.py'))] + return source_fingerprint(paths, package) + + +def read(path): + return json.loads(Path(path).read_text()) + + +def input_path(root, relative): + path = (Path(root) / relative).resolve() + if not path.is_relative_to(Path(root).resolve()): + raise ValueError(f'Input path escapes the mounted input root: {relative}') + return path diff --git a/dicee/query_answering/_deferred.py b/dicee/query_answering/_deferred.py new file mode 100644 index 000000000..39d8d2569 --- /dev/null +++ b/dicee/query_answering/_deferred.py @@ -0,0 +1,52 @@ +"""Validity checks that share one host synchronization per query. + +Reading a CUDA boolean on the host waits for every queued kernel. Inside +``deferred_checks`` such conditions are collected and read together when the +block exits; the first failing check raises its original error. Outside the +block, ``require`` checks immediately, so other callers keep eager errors. +""" + +from contextlib import contextmanager +from contextvars import ContextVar + +import torch + +_PENDING = ContextVar('dicee_pending_checks', default=None) + + +def require(condition, error): + """Raise ``error`` unless the scalar ``condition`` holds.""" + pending = _PENDING.get() + if pending is None or not isinstance(condition, torch.Tensor) or condition.device.type == 'cpu': + if not bool(condition): + raise error + else: + pending.append((condition.reshape(()), error)) + + +def _raise_first(pending): + if pending: + device = pending[0][0].device + flags = torch.stack([condition.to(device) for condition, _ in pending]).tolist() + for passed, (_, error) in zip(flags, pending): + if not passed: + raise error + + +@contextmanager +def deferred_checks(): + """Collect device conditions; an earlier failed check takes precedence over later errors.""" + pending = [] + token = _PENDING.set(pending) + try: + yield + except Exception: + # Interrupts and exits propagate unchanged. + _PENDING.reset(token) + token = None + _raise_first(pending) + raise + finally: + if token is not None: + _PENDING.reset(token) + _raise_first(pending) diff --git a/dicee/query_answering/_query.py b/dicee/query_answering/_query.py new file mode 100644 index 000000000..79c36ba27 --- /dev/null +++ b/dicee/query_answering/_query.py @@ -0,0 +1,240 @@ +"""One query grammar and composition algebra for inference and supervision.""" + +import torch + +from .catalog import PLUS_H_SHAPES, QUERY_SHAPES, ULTRAQUERY_SHAPES # noqa: F401 (re-exported) + + +def nested(value): + return tuple(nested(v) for v in value) if isinstance(value, (tuple, list)) else value + + +def index_query(query_type, query, entities, relations): + """Translate named queries by structure, so operator-like names remain valid IDs.""" + if query_type not in QUERY_SHAPES: + raise ValueError(f'Unknown query type {query_type!r}; choose from {sorted(QUERY_SHAPES)}') + + def visit(pattern, value): + if pattern in ('e', 'r'): + mapping = entities if pattern == 'e' else relations + if value not in mapping: + raise ValueError(f'Unknown {"entity" if pattern == "e" else "relation"}: {value!r}') + return mapping[value] + if pattern == 'n': + if value not in ('n', 'not', -2): + raise ValueError('Expected a negation marker (not, n, or -2)') + return -2 + if pattern == 'u': + if value not in ('u', 'union', 'or', -1): + raise ValueError('Expected a union marker (union, u, or -1)') + return -1 + # The named API can infer union from query_type alone. + if isinstance(value, (tuple, list)) and pattern[-1] == ('u',) and len(value) == len(pattern) - 1: + value = (*value, ('union',)) + if not isinstance(value, (tuple, list)) or len(pattern) != len(value): + raise ValueError(f'Query does not match {query_type}: expected {pattern!r}') + return tuple(visit(p, v) for p, v in zip(pattern, value)) + + return visit(QUERY_SHAPES[query_type], query) + + +def compile_query(query): + query = nested(query) + if type(query) is int and query >= 0: + return ('anchor', query) + if not isinstance(query, tuple) or not query: + raise ValueError('Expected an indexed structured query') + if len(query) == 2 and isinstance(query[1], tuple) and query[1] and all(type(r) is int for r in query[1]) and query[1] != (-1,): + node = compile_query(query[0]) + for relation in query[1]: + if relation == -2: + node = ('not', node) + elif relation >= 0: + node = ('project', relation, node) + else: + raise ValueError('Unknown relation/negation marker') + return node + union = query[-1] == (-1,) + branches = query[:-1] if union else query + if len(branches) < 2: + raise ValueError('Composition requires at least two branches') + return ('or' if union else 'and', *(compile_query(q) for q in branches)) + + +def validate_tree(node, n, nr): + if node[0] == 'anchor': + if not 0 <= node[1] < n: + raise ValueError('Anchor outside the entity vocabulary') + elif node[0] == 'project': + if not 0 <= node[1] < nr: + raise ValueError('Relation outside the relation vocabulary') + validate_tree(node[2], n, nr) + else: + for child in node[1:]: + validate_tree(child, n, nr) + + +def positive(node): + if node[0] == 'not': + return False + return all(positive(child) for child in node[1:] if isinstance(child, tuple)) + + +def atomic_conditions(node): + if node[0] == 'project': + if node[2][0] == 'anchor': + yield node[2][1], node[1] + else: + yield from atomic_conditions(node[2]) + elif node[0] != 'anchor': + for child in node[1:]: + yield from atomic_conditions(child) + + +def relation_signature(node): + if node[0] == 'anchor': + return ('anchor',) + if node[0] == 'project': + return ('project', node[1], relation_signature(node[2])) + return (node[0], *(relation_signature(child) for child in node[1:])) + + +def stable_topk(values, k): + """Select descending scores with entity-ID ties and a CPU partial sort.""" + k = min(k, len(values)) + if values.is_cuda or k == len(values): + return values.argsort(descending=True, stable=True)[:k] + threshold = values.topk(k, sorted=False).values.min() + better = (values > threshold).nonzero().flatten() + tied = (values == threshold).nonzero().flatten()[:k - len(better)] + selected = torch.cat((better, tied)).sort().values + return selected[values[selected].argsort(descending=True, stable=True)] + + +def exact_answers(node, outgoing, num_entities): + """Finite-domain set semantics shared by observed proofs and query generation.""" + op = node[0] + if op == 'anchor': + return {node[1]} + if op == 'project': + return set().union(*(outgoing.get(h, {}).get(node[1], set()) for h in exact_answers(node[2], outgoing, num_entities))) + if op == 'not': + return set(range(num_entities)) - exact_answers(node[1], outgoing, num_entities) + if op == 'and' and any(c[0] != 'not' for c in node[1:]): + included = [exact_answers(c, outgoing, num_entities) for c in node[1:] if c[0] != 'not'] + result = set.intersection(*included) + for child in node[1:]: + if child[0] == 'not': + result.difference_update(exact_answers(child[1], outgoing, num_entities)) + return result + branches = [exact_answers(c, outgoing, num_entities) for c in node[1:]] + return set.intersection(*branches) if op == 'and' else set.union(*branches) + + +def log_complement(value): + """Stable log(1-exp(value)), including exact zero/one memberships. + + Masked evaluation avoids undefined derivatives in inactive torch.where branches. + Without gradients on CUDA, selection applies the same per-element operations + without the host synchronization required by boolean indexing. (CPU kernels + may finish array tails with scalar code, so CPU keeps the masked subsets.) + """ + if value.is_cuda and not (torch.is_grad_enabled() and value.requires_grad): + interior = (value < 0) & torch.isfinite(value) + result = torch.where(value < -0.6931471805599453, torch.log1p(-value.exp()), torch.log(-torch.expm1(value))) + result = torch.where(interior, result, torch.nan) + result = torch.where(value == 0, -torch.inf, result) + return torch.where(torch.isneginf(value), 0., result) + result = torch.empty_like(value) + result[value == 0] = -torch.inf + result[torch.isneginf(value)] = 0. + interior = (value < 0) & torch.isfinite(value) + low = interior & (value < -0.6931471805599453) + result[low] = torch.log1p(-value[low].exp()) + high = interior & ~low + result[high] = torch.log(-torch.expm1(value[high])) + result[~(interior | (value == 0) | torch.isneginf(value))] = torch.nan + return result + + +def combine(values, operator, tnorm='prod', *, logits=False): + values = torch.stack(values) + if logits: + if operator == 'and': + return values.prod(0) if tnorm == 'prod' else values.min(0).values + return 1 - (1 - values).prod(0) if tnorm == 'prod' else values.max(0).values + if operator == 'and': + return values.sum(0) if tnorm == 'prod' else values.min(0).values + return log_complement(log_complement(values).sum(0)) if tnorm == 'prod' else values.max(0).values + + +def negate(value, norm='standard', parameter=0., *, logits=False): + if not logits and norm == 'standard': + return log_complement(value) + p = value if logits else value.exp() + if norm == 'standard': + result = 1 - p + elif norm == 'sugeno': + result = (1 - p) / (1 + parameter * p) + else: + result = (1 - p.pow(parameter)).clamp_min(0).pow(1 / parameter) + return result if logits else result.log() + + +def execute_query(tree, row_batches, *, n, device, row_batch_size, beam_size=64, + tnorm="prod", neg_norm="standard", lambda_=0., use_logits=False, executor="cqd", stats=None, skip_zero_prefix=False, + negation_answers=None): + """Compose rows with live beam selection; gradients flow through retained scores. + + With ``negation_answers`` (a callable from a subtree to its exact answers over the known facts), a negated branch + excludes exactly those answers and nothing else: membership zero on them, one elsewhere. + """ + if stats is None: + stats = dict(negated_pruning=False, bound_skipped=0) + def execute(node): + op = node[0] + if op == 'anchor': + result = torch.full((n,), 0. if use_logits else -torch.inf, dtype=torch.float64, device=device) + result[node[1]] = 1. if use_logits else 0. + return result, False + if op == 'project' and node[2][0] == 'anchor': + return next(row_batches([node[2][1]], node[1]))[1][0], False + if op in ('and', 'or'): + branches = [execute(child) for child in node[1:]] + return combine([v for v, _ in branches], op, tnorm, logits=use_logits), any(p for _, p in branches) + if op == 'not' and negation_answers is not None: + result = torch.zeros((n,), dtype=torch.float64, device=device) + answers = sorted(negation_answers(node[1])) + if answers: + result[torch.tensor(answers, device=device)] = -torch.inf + return result, False + if op == 'not': + values, pruned = execute(node[1]) + stats['negated_pruning'] |= pruned + return negate(values, neg_norm, lambda_, logits=use_logits), pruned + prefix, pruned = execute(node[2]) + top = stable_topk(prefix, n if executor == 'qto' else beam_size) + skip = skip_zero_prefix and executor == 'cqd' and not use_logits and not torch.is_grad_enabled() + # One host copy provides the beam and, when needed, its zero-membership flags. + heads, empty = torch.stack((top, torch.isneginf(prefix[top]).long())).tolist() if skip else (top.tolist(), None) + result = torch.full_like(prefix, -torch.inf) + for start in range(0, len(heads), row_batch_size): + # Edge memberships are at most one: remaining prefixes bound every tail. + if executor == 'qto' and (result >= prefix[heads[start]]).all().item(): + stats['bound_skipped'] += len(heads) - start + break + selected = heads[start:start + row_batch_size] + # Keep mixed batches intact: neural scores can depend on batch geometry. + if skip and all(empty[start:start + row_batch_size]): + stats['zero_prefix_skipped'] = stats.get('zero_prefix_skipped', 0) + len(selected) + continue + batch, edges = next(row_batches(selected, node[1])) + # The device beam indexes the same rows without a host-to-device copy. + values = prefix[top[start:start + len(batch)] if batch == selected else batch, None] + if tnorm == 'min': + joined = torch.minimum(values, edges) + else: + joined = values * edges if use_logits else values + edges + result = torch.maximum(result, joined.amax(dim=0)) + return result, pruned or (executor == 'cqd' and beam_size < n) + return execute(tree) diff --git a/dicee/query_answering/_rank_trace.py b/dicee/query_answering/_rank_trace.py new file mode 100644 index 000000000..88ca257ed --- /dev/null +++ b/dicee/query_answering/_rank_trace.py @@ -0,0 +1,46 @@ +"""Transactional rank records for inference-free tie-policy analysis.""" + +import json +import sqlite3 +from pathlib import Path + +from .context import fingerprint + + +class RankTrace: + """One row per query; answer records are [entity, rank, greater, tie_size].""" + + def __init__(self, path, identity, completed): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + self.connection = sqlite3.connect(path) + try: + self.connection.execute('PRAGMA synchronous=FULL') + self.connection.execute('CREATE TABLE IF NOT EXISTS metadata (identity TEXT NOT NULL)') + self.connection.execute('CREATE TABLE IF NOT EXISTS queries (position INTEGER PRIMARY KEY, query_id TEXT UNIQUE, shape TEXT, answers TEXT)') + rows = self.connection.execute('SELECT identity FROM metadata').fetchall() + if rows and rows != [(fingerprint(identity),)]: + raise ValueError('Rank trace identity changed') + if not rows: + self.connection.execute('INSERT INTO metadata VALUES (?)', (fingerprint(identity),)) + count, last = self.connection.execute('SELECT COUNT(*), MAX(position) FROM queries WHERE position < ?', (completed,)).fetchone() + if count != completed or (completed and last != completed - 1): + raise ValueError('Rank trace is missing checkpointed queries') + self.connection.execute('DELETE FROM queries WHERE position >= ?', (completed,)) + self.connection.commit() + except BaseException: + self.connection.close() + raise + + def add(self, position, query, answers): + self.connection.execute('INSERT INTO queries VALUES (?, ?, ?, ?)', + (position, query.identity, query.shape, json.dumps(answers, separators=(',', ':')))) + + def commit(self): + self.connection.commit() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.connection.close() diff --git a/dicee/query_answering/adapter_training.py b/dicee/query_answering/adapter_training.py new file mode 100644 index 000000000..ad0cfb403 --- /dev/null +++ b/dicee/query_answering/adapter_training.py @@ -0,0 +1,675 @@ +"""Source-only adapter fitting with frozen backbone score banks. + +Fits the shared inference transform using a filtered-softmax loss. +""" + +import copy +import json +import math +import os +import random +import time +from contextlib import ExitStack +from dataclasses import dataclass, field +from pathlib import Path + +import torch + +from ._checkpoint import implementation_fingerprint, scoring_fingerprint +from ._query import QUERY_SHAPES, ULTRAQUERY_SHAPES, atomic_conditions, compile_query, execute_query, nested, positive, validate_tree +from .context import QueryContext, attached_context, fingerprint, state_fingerprint +from .engine import AtomicScorer +from .score_adapter import QueryScoreAdapter + +FLAT_SHAPES = ('2i', '3i', '2in', '3in') +TRAINING_SHAPES = ULTRAQUERY_SHAPES +STANDARD_TRAINING_SHAPES = ('1p', '2p', '3p', '2i', '3i', '2in', '3in', 'inp', 'pin', 'pni') + + +def _canonical_query(query): + if isinstance(query, (int, str)): + return query + if len(query) == 2 and all(isinstance(v, int) or v in ('r', 'n') for v in query[1]) and query[1] not in ((-1,), ('u',)): + return _canonical_query(query[0]), tuple(query[1]) + union = query[-1] in ((-1,), ('u',)) + branches = [_canonical_query(q) for q in (query[:-1] if union else query)] + if all(len(q) == 2 and isinstance(q[0], int) for q in branches): + branches.sort(key=lambda q: (len(q[1]), q)) + else: + branches.sort(key=repr) + return (*branches, query[-1]) if union else tuple(branches) + + +def _structure(query): + if isinstance(query, int): + return 'e' + if len(query) == 2 and all(isinstance(v, int) for v in query[1]) and query[1] != (-1,): + return _structure(query[0]), tuple('n' if r == -2 else 'r' for r in query[1]) + union = query[-1] == (-1,) + branches = tuple(sorted((_structure(q) for q in (query[:-1] if union else query)), key=repr)) + return (*branches, ('u',)) if union else branches + + +_SHAPE_NAMES = {_canonical_query(QUERY_SHAPES[s]): s for s in TRAINING_SHAPES} + + +@dataclass(frozen=True) +class AdapterQuery: + """A source query, its complete answers, and optionally the hard answers it is trained on. + + ``positives`` defaults to every hard answer (answers not provable from the + masked context). Other answers are filtered from the ranking either way. + """ + + query: tuple + answers: frozenset + positives: frozenset | None = None + + def __post_init__(self): + query = _canonical_query(nested(self.query)) + tree = compile_query(query) + shape = _SHAPE_NAMES.get(_structure(query)) + if shape is None: + raise ValueError('Unsupported adapter training query structure') + conditions = list(atomic_conditions(tree)) + if shape in FLAT_SHAPES and len(set(conditions)) != len(conditions): + raise ValueError('Training intersections require distinct atoms') + object.__setattr__(self, 'query', query) + object.__setattr__(self, 'answers', frozenset(self.answers)) + if self.positives is not None: + object.__setattr__(self, 'positives', frozenset(self.positives)) + + @property + def shape(self): + return _SHAPE_NAMES[_structure(self.query)] + + @property + def conditions(self): + return list(atomic_conditions(compile_query(self.query))) + + +@dataclass(frozen=True) +class AdapterTrainingData: + """Prepared context and complete source answer sets, with disjoint query splits.""" + + name: str + context: QueryContext + train: tuple + validation: tuple = () + metadata: dict = field(default_factory=dict) + + def __post_init__(self): + if not self.name or not self.train: + raise ValueError('A named source and nonempty training split are required') + seen = set() + for item in (*self.train, *self.validation): + tree = compile_query(item.query) + validate_tree(tree, self.context.num_entities, self.context.num_relations) + if item.query in seen: + raise ValueError('Training/validation queries must be distinct, including branch permutations') + seen.add(item.query) + if any(type(a) is not int or not 0 <= a < self.context.num_entities for a in item.answers): + raise ValueError('Answer outside the source vocabulary') + easy = self.context.answers(tree) + if ('n' not in item.shape and not easy <= item.answers) or not item.answers - easy: + raise ValueError('Source answers must contain hard answers and all positive context proofs') + if item.positives is not None and (not item.positives or not item.positives <= item.answers - easy): + raise ValueError('Positives must be a nonempty subset of the hard answers') + if len(item.answers) == self.context.num_entities: + raise ValueError('Training queries need at least one negative candidate') + + def to_dict(self): + def records(items): + return [dict(query=q.query, answers=sorted(q.answers), + **({} if q.positives is None else dict(positives=sorted(q.positives)))) for q in items] + return dict(version=1, name=self.name, context=self.context.to_dict(), train=records(self.train), + validation=records(self.validation), metadata=self.metadata) + + def save(self, path): + Path(path).write_text(json.dumps(self.to_dict(), indent=2) + '\n') + + @classmethod + def load(cls, path): + data = json.loads(Path(path).read_text()) + if data.get('version') != 1: + raise ValueError('Unsupported adapter data version') + return cls(data['name'], QueryContext(**data['context']), + tuple(AdapterQuery(**q) for q in data['train']), + tuple(AdapterQuery(**q) for q in data.get('validation', [])), data.get('metadata', {})) + + +def answer_levels(tree, full, observed): + """Minimum number of unobserved links each answer needs, counted along positive branches. + + Intersections add their branches' counts and unions take the cheapest + branch. Negated branches restrict answers by ``full`` truth without adding + links. Answers provable from ``observed`` alone have level 0. + """ + op = tree[0] + if op == 'anchor': + return {tree[1]: 0} + if op == 'project': + result = {} + for head, cost in answer_levels(tree[2], full, observed).items(): + seen = observed.outgoing.get(head, {}).get(tree[1], ()) + for tail in full.outgoing.get(head, {}).get(tree[1], ()): + value = cost + (tail not in seen) + if value < result.get(tail, value + 1): + result[tail] = value + return result + if op == 'or': + result = {} + for child in tree[1:]: + for answer, value in answer_levels(child, full, observed).items(): + result[answer] = min(result.get(answer, value), value) + return result + if op == 'and' and any(child[0] != 'not' for child in tree[1:]): + parts = [answer_levels(child, full, observed) for child in tree[1:] if child[0] != 'not'] + result = {a: sum(part[a] for part in parts) for a in min(parts, key=len) if all(a in part for part in parts)} + for child in tree[1:]: + if child[0] == 'not': + for answer in full.answers(child[1]): + result.pop(answer, None) + return result + raise ValueError('Answer levels need a positive anchored branch') + + +def _max_level(tree): + if tree[0] in ('anchor', 'not'): + return 0 + if tree[0] == 'project': + return 1 + _max_level(tree[2]) + levels = [_max_level(child) for child in tree[1:]] + return sum(levels) if tree[0] == 'and' else max(levels) + + +def _shape_tree(shape): + """Compile a query type's pattern with placeholder IDs.""" + def visit(pattern): + if pattern in ('e', 'r', 'n', 'u'): + return {'e': 0, 'r': 0, 'n': -2, 'u': -1}[pattern] + return tuple(visit(part) for part in pattern) + return compile_query(visit(QUERY_SHAPES[shape])) + + +def _split(total, parts): + return [total // parts + (i < total % parts) for i in range(parts)] + + +def prepare_adapter_data(source, *, name='source', mask_fraction=.3, train_per_shape=96, + validation_per_shape=32, seed=2026090851, max_attempts=100_000, shapes=('2i', '3i'), + extend=None, train_counts=None, hardness='standard'): + """Mask reciprocal pairs and reuse QueryGenerator's grounding on source facts. + + ``source`` must contain source-training facts only. Never pass target test + facts here. Answer supervision uses the source; scoring sees only the mask. + + ``hardness='balanced'`` gives each query a level in rotation, from 1 to the + type's maximum, and trains it on its hard answers at that level (see + ``answer_levels``), so every level gets the same number of queries. + """ + from ..query_generator import QueryGenerator + if not 0 < mask_fraction < 1 or train_per_shape < 1 or validation_per_shape < 0 or max_attempts < 1: + raise ValueError('Require 0 < mask_fraction < 1 and positive training/attempt counts') + if hardness not in ('standard', 'balanced') or (hardness == 'balanced' and extend is not None): + raise ValueError("Choose hardness 'standard' or 'balanced'; only standard data can be extended") + shapes = tuple(shapes) + if not shapes or len(set(shapes)) != len(shapes) or any(s not in TRAINING_SHAPES for s in shapes): + raise ValueError(f'Choose distinct training shapes from {TRAINING_SHAPES}') + counts = {s: train_per_shape for s in shapes} if train_counts is None else dict(train_counts) + if set(counts) != set(shapes) or any(type(n) is not int or n < 1 for n in counts.values()): + raise ValueError('Positive training counts required for every prepared shape') + rng = random.Random(seed) + inverse = {a: b for h, t in source.inverse_relations for a, b in ((h, t), (t, h))} + def canonical(edge): + h, r, t = edge + return min(edge, (t, inverse[r], h)) if r in inverse else edge + pairs = sorted({canonical(edge) for edge in source.triples}) + count = max(1, round(len(pairs) * mask_fraction)) + if count >= len(pairs): + raise ValueError('Source is too small to retain a nonempty masked graph') + hidden = set(rng.sample(pairs, count)) + context = QueryContext([edge for edge in source.triples if canonical(edge) not in hidden], + source.num_entities, source.num_relations, source.inverse_relations) + if extend is not None: + if (extend.name != name or extend.context.identity != context.identity + or extend.metadata.get('source_sha256') != source.identity + or set(q.shape for q in (*extend.train, *extend.validation)) - set(shapes)): + raise ValueError('Extension requires the same source, mask, name, and shapes') + for shape in shapes: + if (sum(q.shape == shape for q in extend.train) > counts[shape] + or sum(q.shape == shape for q in extend.validation) != validation_per_shape): + raise ValueError('Extension must retain all training and validation queries') + generator = QueryGenerator.from_context(source, seed=seed) + training = list(extend.train) if extend is not None else [] + validation = list(extend.validation) if extend is not None else [] + seen = {q.query for q in (*training, *validation)} + for shape in shapes: + width = int(shape[0]) if shape in FLAT_SHAPES else 1 + targets = sorted(t for t, rs in generator.ent_in.items() if sum(map(len, rs.values())) >= width) + levels = _max_level(_shape_tree(shape)) if hardness == 'balanced' else 1 + train_demand, validation_demand = _split(counts[shape], levels), _split(validation_per_shape, levels) + if extend is not None: + train_demand = [counts[shape] - sum(q.shape == shape for q in training)] + demand = [t + (0 if extend is not None else v) for t, v in zip(train_demand, validation_demand)] + pools = [[] for _ in range(levels)] + for _ in range(max_attempts): + if all(len(pool) == n for pool, n in zip(pools, demand)) or not targets: + break + query = generator.tuple2list(QUERY_SHAPES[shape]) + if generator.fill_query(query, generator.ent_in, generator.ent_out, rng.choice(targets)): + continue + query = _canonical_query(nested(query)) + if query in seen: + continue + tree = compile_query(query) + answers = source.answers(tree) + context_answers = context.answers(tree) + if not answers - context_answers or len(answers) == source.num_entities: + continue + if 'n' in shape and not context_answers - answers: + continue + if hardness == 'standard': + seen.add(query) + pools[0].append(AdapterQuery(query, answers)) + continue + hard = answers - context_answers + level = answer_levels(tree, source, context) + if not hard <= level.keys() or not all(1 <= level[a] <= levels for a in hard): + raise RuntimeError(f'Answer levels disagree with exact answers for {shape} {query}') + # Rarer, higher levels first; skip queries whose levels are already filled. + chosen = next((v for v in sorted({level[a] for a in hard}, reverse=True) if len(pools[v - 1]) < demand[v - 1]), None) + if chosen is None: + continue + seen.add(query) + pools[chosen - 1].append(AdapterQuery(query, answers, frozenset(a for a in hard if level[a] == chosen))) + if any(len(pool) != n for pool, n in zip(pools, demand)): + raise ValueError(f'Only generated {sum(map(len, pools))} distinct {shape} queries; reduce counts or supply prepared data') + shape_train, shape_validation = [], [] + for pool, n in zip(pools, train_demand): + rng.shuffle(pool) + shape_train.extend(pool[:n]) + shape_validation.extend(pool[n:]) + if hardness == 'balanced': + rng.shuffle(shape_train) + rng.shuffle(shape_validation) + training.extend(shape_train) + validation.extend(shape_validation) + return AdapterTrainingData(name, context, tuple(training), tuple(validation), + dict(extend.metadata if extend is not None else {}, + source_sha256=source.identity, seed=seed, mask_fraction=mask_fraction, + masked_fact_pairs=count, train_per_shape=train_per_shape, + train_counts=counts, + validation_per_shape=validation_per_shape, shapes=list(shapes), + **({} if hardness == 'standard' else dict(hardness=hardness)))) + + +def _bank(model, data, *, cache_dir, row_batch_size, seed, samples, device='cpu', + cache_bytes=128 * 1024**2, device_cache_bytes=0): + if any(q.shape not in FLAT_SHAPES for q in (*data.train, *data.validation)): + from ._adapter_cache import TrainingRows + provider = TrainingRows(model, data.context, cache_dir=cache_dir, row_batch_size=row_batch_size, + seed=seed, samples=samples, device=device, + cache_bytes=cache_bytes, device_cache_bytes=device_cache_bytes) + return dict(provider=provider, identity=provider.identity, context=data.context) + from ..models._inference import float32_precision_token + conditions = sorted({pair for q in (*data.train, *data.validation) for pair in q.conditions}) + settings = {name: getattr(model, name, None) for name in ( + 'query_batch_size', 'test_samples', 'walk_num', 'walk_len', 'refinements', + 'compact_state', 'compile_sampler', 'pack_walks', 'prefetch_walks', '_inference_backend')} + identity = dict(version=1, backbone=state_fingerprint(model), context=data.context.identity, + conditions=conditions, seed=seed, samples=samples, row_batch_size=row_batch_size, + settings=settings, torch=str(torch.__version__), device=str(next(model.parameters()).device), + precision=float32_precision_token(), deterministic=torch.are_deterministic_algorithms_enabled(), + autocast=torch.is_autocast_enabled(next(model.parameters()).device.type), + scoring_code=scoring_fingerprint(), architecture=str(model), + configuration={k: str(v) for k, v in getattr(model, 'args', {}).items()}) + if hasattr(model, 'prompt_identity'): + identity['prompts'] = model.prompt_identity() + key = fingerprint(identity) + path = Path(cache_dir) / f'{key}.pt' if cache_dir is not None else None + if path is not None and path.exists(): + bank = torch.load(path, map_location='cpu', weights_only=True) + if bank['identity'] != identity or bank['rows_sha256'] != state_fingerprint({'rows': bank['rows']}): + raise ValueError('Corrupt or mismatched adapter score bank') + raw = bank['rows'] + else: + with attached_context(model, data.context): + scorer = AtomicScorer(model, data.context, row_batch_size=row_batch_size, seed=seed, samples=samples) + # Copy each scored batch off the device immediately. + raw = torch.cat([scorer.rows(conditions[i:i + row_batch_size]).cpu() for i in range(0, len(conditions), row_batch_size)]) + if path is not None: + path.parent.mkdir(parents=True, exist_ok=True) + torch.save(dict(identity=identity, rows=raw, rows_sha256=state_fingerprint({'rows': raw})), path) + if raw.shape != (len(conditions), data.context.num_entities) or not torch.isfinite(raw).all(): + raise ValueError('Invalid complete score bank') + observed, base = data.context.features(conditions, device='cpu') + return dict(raw=raw.to(torch.float64), observed=observed, base=base, + indices={pair: i for i, pair in enumerate(conditions)}, identity=key, context=data.context) + + +def _prepared_bank(adapter, bank): + if 'provider' in bank: + return bank + device = adapter.weights.device + if bank.get('prepared_configuration') == adapter.configuration and bank.get('prepared_device') == str(device): + return bank + raw, observed, features = [], [], [] + for start in range(0, len(bank['raw']), 32): + end = start + 32 + r, o, f = adapter.prepare(bank['raw'][start:end], bank['observed'][start:end], bank['base'][start:end]) + raw.append(r) + observed.append(o) + features.append(f) + return dict(bank, prepared_raw=torch.cat(raw).to(device), prepared_observed=torch.cat(observed).to(device), + features=torch.cat(features).to(device), + prepared_configuration=adapter.configuration, prepared_device=str(device)) + + +def _context_answers(bank, query): + cache = bank.setdefault('context_answers', {}) + if query.query not in cache: + cache[query.query] = bank['context'].answers(compile_query(query.query)) + return cache[query.query] + + +def _query_logs(adapter, bank, query): + context = bank['context'] + provider = bank.get('provider') + batch_size = 32 if provider is not None else 8 + + def rows(heads, relation): + for start in range(0, len(heads), batch_size): + batch = heads[start:start + batch_size] + conditions = [(head, relation) for head in batch] + if provider is not None: + raw, observed, features = provider.prepared(adapter, conditions, device=adapter.weights.device) + else: + indices = [bank['indices'][pair] for pair in conditions] + raw, observed, features = (bank[k][indices] for k in ('prepared_raw', 'prepared_observed', 'features')) + yield batch, adapter.transform(raw, observed, features) + + trees = bank.setdefault('trees', {}) + if query.query not in trees: + trees[query.query] = compile_query(query.query) + tree = trees[query.query] + result, _ = execute_query(tree, rows, n=context.num_entities, device=adapter.weights.device, row_batch_size=batch_size, + beam_size=bank.get('beam_size', 64), tnorm=bank.get('tnorm', 'prod')) + if adapter.observed_mix == 1 and positive(tree): + result = result.clone() + result[sorted(_context_answers(bank, query))] = 0. + return result + + +def filtered_softmax_loss(logs, answers, hard_answers): + """Mean log(1 + sum_negative exp(score_negative - score_hard)).""" + negative = torch.ones_like(logs, dtype=torch.bool) + negative[sorted(answers)] = False + if not negative.any() or not hard_answers: + raise ValueError('Loss requires hard answers and negative candidates') + positives = logs[sorted(hard_answers)] + negative_scores = logs[negative] + if torch.isneginf(negative_scores).all(): + # Exact negation can rule out every negative. The loss is then zero; + # logsumexp of all -inf values would introduce undefined gradients. + return positives.sum() * 0. + negative_mass = torch.logsumexp(negative_scores, 0) + return (torch.logaddexp(positives, negative_mass) - positives).mean() + + +def _filtered_ranks(scores, answers, hard): + eligible = torch.ones_like(scores, dtype=torch.bool) + eligible[sorted(answers)] = False + negatives, targets = scores[eligible], scores[sorted(hard)] + if targets.isnan().any() or negatives.isnan().any(): + raise ValueError('Cannot rank NaN prediction scores') + if len(hard) < 8: + better = (negatives[None] > targets[:, None]).sum(1) + tied = (negatives[None] == targets[:, None]).sum(1) + else: + ordered = negatives.sort().values + left = torch.searchsorted(ordered, targets) + right = torch.searchsorted(ordered, targets, right=True) + better, tied = len(ordered) - right, right - left + return (1 + better + tied.to(torch.float64) / 2).tolist() + + +def _validation(adapter, sources, banks): + result = {} + device = banks[0].get('validation_device', adapter.weights.device) if banks else adapter.weights.device + if torch.device(device) != adapter.weights.device: + adapter = copy.deepcopy(adapter).to(device) + with torch.no_grad(): + for source, bank in zip(sources, banks): + bank = _prepared_bank(adapter, bank) + for shape in bank.get('validation_shapes', TRAINING_SHAPES): + metrics = [] + for query in source.validation: + if query.shape != shape: + continue + scores = _query_logs(adapter, bank, query) + hard = query.positives if query.positives is not None else query.answers - _context_answers(bank, query) + ranks = _filtered_ranks(scores, query.answers, hard) + metrics.append([sum(1 / rank for rank in ranks) / len(ranks), + *(sum(rank <= k for rank in ranks) / len(ranks) for k in (1, 3, 10))]) + if metrics: + average = torch.tensor(metrics, dtype=torch.float64).mean(0).tolist() + result[f'{source.name}/{shape}'] = dict(zip(('mrr', 'hits1', 'hits3', 'hits10'), average), queries=len(metrics)) + return result + + +def _baseline_validation(adapter, sources, banks, cache_dir): + from ._checkpoint import write_json + if cache_dir is None: + return _validation(adapter, sources, banks) + identity = fingerprint(dict(adapter=adapter.to_dict(), implementation=implementation_fingerprint(), + cpu_threads=torch.get_num_threads(), + sources=[fingerprint(s.to_dict()) for s in sources], + banks=[b['identity'] for b in banks], + execution=[dict(beam=b.get('beam_size', 64), tnorm=b.get('tnorm', 'prod'), shapes=b.get('validation_shapes', TRAINING_SHAPES), + device=str(b.get('validation_device', b.get('device', 'cpu')))) for b in banks])) + path = Path(cache_dir)/f'validation-{identity}.json' + if path.exists(): + saved = json.loads(path.read_text()) + if saved['identity'] != identity or saved['sha256'] != fingerprint(saved['metrics']): + raise ValueError('Corrupt baseline validation cache') + return saved['metrics'] + metrics = _validation(adapter, sources, banks) + write_json(path, dict(identity=identity, metrics=metrics, sha256=fingerprint(metrics))) + return metrics + + +@dataclass +class AdapterFitResult: + adapter: QueryScoreAdapter + history: list + validation: dict + + +def _save_fit(path, state): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + '.tmp') + with temporary.open('wb') as stream: + torch.save(state, stream) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + + +def fit_query_adapter(model, sources, *, feature_mode='context_scores', observed_mix=1., epochs=20, batch_size=8, + learning_rate=.02, identity_penalty=.001, gradient_clip=1., seed=2026090851, + cache_dir=None, row_batch_size=8, samples=None, bias_bound=4., normalization='none', + hidden_dim=0, scale_bound=2., beam_size=64, validation_every=None, validation_sources=None, on_epoch=None, + train_shapes=None, training_sources=None, train_per_shape=None, + early_stopping_patience=None, validation_shapes=None, training_device='cpu', + training_cache_bytes=1536 * 1024**2, device_cache_bytes=512 * 1024**2, + checkpoint_path=None, validation_device=None, tnorm='prod'): + """Fit a frozen-backbone adapter; optionally select epochs on source validation.""" + sources = list(sources) + if tnorm not in ('prod', 'min'): + raise ValueError('Choose prod or min t-norm') + device = torch.device(training_device) + validation_device = torch.device(validation_device or device) + cache_device = device if device.type == 'cuda' else validation_device + if any(d.type not in ('cpu', 'cuda') for d in (device, validation_device)) or min(training_cache_bytes, device_cache_bytes) < 0: + raise ValueError('Choose CPU/CUDA training and nonnegative cache budgets') + if not sources or len({d.name for d in sources}) != len(sources): + raise ValueError('Provide nonempty, uniquely named sources') + if min(epochs, batch_size, row_batch_size) < 1 or not all(math.isfinite(v) for v in (learning_rate, identity_penalty, gradient_clip)): + raise ValueError('Positive epoch/batch counts and finite optimizer settings required') + if learning_rate <= 0 or identity_penalty < 0 or gradient_clip <= 0: + raise ValueError('Invalid optimizer settings') + if validation_every is not None and (type(validation_every) is not int or validation_every < 1): + raise ValueError('validation_every must be a positive integer') + if early_stopping_patience is not None: + if type(early_stopping_patience) is not int or early_stopping_patience < 1: + raise ValueError('early_stopping_patience must be a positive number of epochs') + if validation_every is None: + raise ValueError('Early stopping requires source validation checkpoint selection') + train_shapes = tuple(train_shapes or TRAINING_SHAPES) + if any(s not in TRAINING_SHAPES for s in train_shapes): + raise ValueError('Unsupported training shape') + validation_shapes = tuple(validation_shapes or TRAINING_SHAPES) + if not validation_shapes or any(s not in TRAINING_SHAPES for s in validation_shapes): + raise ValueError('Unsupported validation shape') + if type(beam_size) is not int or beam_size < 1: + raise ValueError('Positive beam_size required') + training_names = set(training_sources) if training_sources is not None else {s.name for s in sources} + if not training_names or training_names - {s.name for s in sources}: + raise ValueError('Choose existing training sources') + if train_per_shape is not None and (type(train_per_shape) is not int or train_per_shape < 1): + raise ValueError('train_per_shape must be a positive integer') + feature_mode = feature_mode or 'context_scores' + with ExitStack() as resources: + backbone = state_fingerprint(model) + banks = [] + total_entities = sum(data.context.num_entities for data in sources) + for data in sources: + bank = _bank(model, data, cache_dir=cache_dir, row_batch_size=row_batch_size, seed=seed, samples=samples, + device=cache_device, cache_bytes=training_cache_bytes * data.context.num_entities // total_entities, + device_cache_bytes=device_cache_bytes * data.context.num_entities // total_entities) + banks.append(bank) + if 'provider' in bank: + resources.callback(bank['provider'].close) + for bank in banks: + bank.update(beam_size=beam_size, tnorm=tnorm, validation_shapes=validation_shapes, device=device, validation_device=validation_device) + selection_names = set(validation_sources) if validation_sources is not None else {s.name for s in sources} + validation_indices = [i for i, s in enumerate(sources) if s.name in selection_names and s.validation] + if validation_every is not None and (not validation_indices or selection_names - {s.name for s in sources}): + raise ValueError('Checkpoint selection requires reserved source validation queries') + settings = dict(objective='filtered_softmax_v1', epochs=epochs, batch_size=batch_size, learning_rate=learning_rate, + identity_penalty=identity_penalty, gradient_clip=gradient_clip, seed=seed, + weighting='equal_source_shape', selection='source_validation_mrr' if validation_every else 'final_epoch', + shapes=sorted({q.shape for s in sources for q in s.train if q.shape in train_shapes}), + validation_sources=sorted(selection_names), validation_every=validation_every, + training_sources=sorted(training_names), train_per_shape=train_per_shape, + early_stopping_patience=early_stopping_patience, beam_size=beam_size, tnorm=tnorm, validation_shapes=list(validation_shapes), + device=str(device), validation_device=str(validation_device), cpu_threads=torch.get_num_threads(), training_cache_bytes=training_cache_bytes, + device_cache_bytes=device_cache_bytes) + adapter = QueryScoreAdapter(feature_mode, observed_mix, metadata=dict( + backbone_state_sha256=backbone, training=settings, score_banks=[bank['identity'] for bank in banks], + sources={source.name: fingerprint(source.to_dict()) for source in sources}), bias_bound=bias_bound, scale_bound=scale_bound, + normalization=normalization, hidden_dim=hidden_dim, seed=seed).to(device) + prepared = [_prepared_bank(adapter, bank) for bank in banks] + optimizer = torch.optim.Adam(adapter.parameters(), lr=learning_rate) + examples = [] + selected_counts: dict[tuple[int, str], int] = {} + for i, source in enumerate(sources): + if source.name not in training_names: + continue + for query in source.train: + cell = (i, query.shape) + if query.shape in train_shapes and (train_per_shape is None or selected_counts.get(cell, 0) < train_per_shape): + examples.append((i, query)) + selected_counts[cell] = selected_counts.get(cell, 0) + 1 + if not examples: + raise ValueError('No queries for the selected training shapes') + hard_answers = [q.positives if q.positives is not None else q.answers - sources[i].context.answers(compile_query(q.query)) + for i, q in examples] + cells = {(i, q.shape) for i, q in examples} + counts = {cell: sum((i, q.shape) == cell for i, q in examples) for cell in cells} + rng, history, best = random.Random(seed), [], None + stopped_early = False + resume_identity = fingerprint(dict(adapter=adapter.to_dict(), implementation=implementation_fingerprint(), + learning_rate=learning_rate, identity_penalty=identity_penalty, + gradient_clip=gradient_clip, seed=seed)) + if checkpoint_path is not None and Path(checkpoint_path).exists(): + saved = torch.load(checkpoint_path, map_location='cpu', weights_only=True) + if saved['identity'] != resume_identity: + raise ValueError('Adapter checkpoint inputs/settings changed') + adapter.load_state_dict(saved['adapter']) + optimizer.load_state_dict(saved['optimizer']) + rng.setstate(saved['rng']) + history, best = saved['history'], saved['best'] + stopped_early = history[-1].get('early_stopped', False) if history else False + for epoch in range(epochs if stopped_early else len(history), epochs): + if device.type == 'cuda': + torch.cuda.synchronize(device) + started = time.perf_counter() + order = list(range(len(examples))) + rng.shuffle(order) + epoch_loss = 0. + for start in range(0, len(order), batch_size): + optimizer.zero_grad() + batch = order[start:start + batch_size] + batch_terms = [] + for index in batch: + i, query = examples[index] + hard = hard_answers[index] + logs = _query_logs(adapter, prepared[i], query) + weight = len(examples) / (len(cells) * counts[i, query.shape]) + term = weight * filtered_softmax_loss(logs, query.answers, hard) / len(batch) + term.backward() + batch_terms.append(term.detach()) + terms = torch.stack(batch_terms).tolist() + if not all(math.isfinite(term) for term in terms): + raise FloatingPointError('Nonfinite adapter training objective') + batch_loss = sum(terms) + penalty = torch.cat([p.reshape(-1) for p in adapter.parameters()]).square().mean() + loss = identity_penalty * penalty + if not torch.isfinite(loss): + raise FloatingPointError('Nonfinite adapter training objective') + loss.backward() + torch.nn.utils.clip_grad_norm_(adapter.parameters(), gradient_clip, error_if_nonfinite=True) + optimizer.step() + epoch_loss += batch_loss + float(loss.detach()) + if device.type == 'cuda': + torch.cuda.synchronize(device) + record = dict(epoch=epoch + 1, loss=epoch_loss / math.ceil(len(examples) / batch_size), + train_seconds=time.perf_counter() - started) + if validation_every and ((epoch + 1) % validation_every == 0 or epoch + 1 == epochs): + started = time.perf_counter() + metrics = _validation(adapter, [sources[i] for i in validation_indices], [prepared[i] for i in validation_indices]) + record['validation_seconds'] = time.perf_counter() - started + record['validation_mrr'] = sum(v['mrr'] for v in metrics.values()) / len(metrics) + if not math.isfinite(record['validation_mrr']): + raise FloatingPointError('Nonfinite source-validation MRR') + if best is None or record['validation_mrr'] > best[0]: + best = (record['validation_mrr'], epoch + 1, + {k: v.detach().clone() for k, v in adapter.state_dict().items()}, metrics) + if early_stopping_patience is not None: + record.update(best_epoch=best[1], epochs_without_improvement=epoch + 1 - best[1]) + stopped_early = epoch + 1 < epochs and epoch + 1 - best[1] >= early_stopping_patience + record['early_stopped'] = stopped_early + history.append(record) + if checkpoint_path is not None and ((epoch + 1) % (validation_every or 1) == 0 or epoch + 1 == epochs): + _save_fit(checkpoint_path, dict(identity=resume_identity, adapter=adapter.state_dict(), + optimizer=optimizer.state_dict(), rng=rng.getstate(), history=history, best=best)) + if on_epoch is not None: + on_epoch(record) + if stopped_early: + break + if best is not None: + adapter.load_state_dict(best[2]) + settings.update(selected_epoch=best[1], selected_validation_mrr=best[0]) + settings.update(epochs_completed=len(history), stopped_early=stopped_early, + stop_reason='validation_patience' if stopped_early else 'epoch_ceiling', + training_queries=len(examples), optimizer_steps=len(history) * math.ceil(len(examples) / batch_size)) + if state_fingerprint(model) != backbone: + raise RuntimeError('Backbone changed during adapter fitting') + settings['row_cache'] = {source.name: dict(bank['provider'].stats) for source, bank in zip(sources, banks) if 'provider' in bank} + variants = {'sigmoid': QueryScoreAdapter('global').to(device), 'observed': QueryScoreAdapter('global', 1.).to(device)} + validation = {name: _baseline_validation(value, sources, banks, cache_dir) for name, value in variants.items()} + validation['fitted'] = best[3] if best is not None and len(validation_indices) == len(sources) else _validation(adapter, sources, banks) + return AdapterFitResult(adapter, history, validation) diff --git a/dicee/query_answering/benchmark.py b/dicee/query_answering/benchmark.py new file mode 100644 index 000000000..d430c0270 --- /dev/null +++ b/dicee/query_answering/benchmark.py @@ -0,0 +1,667 @@ +"""Streaming, full-candidate evaluation of published logical queries. + +The default sort/filter/query/type averaging protocol follows UltraQuery's +query_utils.py at 427966ad8ed60420eef034063d44f3153addff90. Predictions are +provided independently of labels so any query executor can use this evaluator. +""" + +import hashlib +import heapq +import json +import time +from bisect import bisect_right +from collections import Counter, defaultdict +from collections.abc import Callable, Mapping, Sequence +from contextlib import ExitStack, nullcontext +from pathlib import Path +from typing import Any + +import torch + +from ..models._inference import to_device +from ._checkpoint import BenchmarkCheckpoint, implementation_fingerprint, write_json +from ._query import QUERY_SHAPES, ULTRAQUERY_SHAPES, compile_query, relation_signature +from ._rank_trace import RankTrace +from .context import attached_context, evaluation_mode, fingerprint, state_fingerprint +from .datasets import BENCHMARK_DATASETS, PLUS_H_DATASETS, BenchmarkQuery, QueryBenchmark, load_benchmark +from .engine import QueryAnswerer +from .score_adapter import QueryScoreAdapter + +METRICS = ('mrr', 'hits1', 'hits3', 'hits10') + + +def _accumulate_inference_statistics(stats, info, engines): + """Persist event counts and the combined calibrated-row cache occupancy.""" + for key in ('raw_rows', 'cache_hits', 'pruned', 'negated_pruning', 'bound_skipped', 'zero_prefix_skipped'): + stats[key] += int(info.get(key, 0)) + stats['cache_bytes'] = sum(engine._cache_used for engine in engines) + stats['cache_peak_bytes'] = max(stats.get('cache_peak_bytes', 0), stats['cache_bytes']) + + +class QueryMetrics: + """Reuse a validated candidate domain and device masks across a dataset.""" + + def __init__(self, num_entities, *, candidates=None, tie_policy='sort'): + if tie_policy not in ('sort', 'average', 'optimistic', 'pessimistic', 'expected'): + raise ValueError('Unknown benchmark tie policy') + self.n, self.tie_policy = num_entities, tie_policy + domain = range(num_entities) if candidates is None else frozenset(candidates) + if not domain or min(domain) < 0 or max(domain) >= num_entities: + raise ValueError('Invalid candidate domain') + self.full_domain = len(domain) == num_entities + self.domain = range(num_entities) if self.full_domain else domain + self.outside = frozenset() if self.full_domain else frozenset(range(num_entities)) - domain + self._device = None + self._harmonic = None + + def __call__(self, scores, easy, hard): + return self.evaluate(scores, easy, hard, (self.tie_policy,))[0][self.tie_policy] + + def evaluate(self, scores, easy, hard, policies=('sort', 'expected'), *, records=False): + """Share one argsort across tie policies and optional answer-rank records.""" + if not policies or set(policies) - {'sort', 'expected', 'average', 'optimistic', 'pessimistic'}: + raise ValueError('Unknown benchmark tie policy') + if scores.ndim != 1 or len(scores) != self.n: + raise ValueError('Expected one complete higher-is-better score vector') + # Read with the metrics below: one device-to-host copy per evaluation. + invalid = (torch.isnan(scores) | torch.isposinf(scores)).any() + easy, hard = frozenset(easy), frozenset(hard) + answers = easy | hard + if not hard or easy & hard or any(t not in self.domain for t in answers): + if invalid.item(): + raise ValueError('Expected one complete higher-is-better score vector') + raise ValueError('Need disjoint easy/hard answers and nonempty hard answers inside the candidate domain') + targets = sorted(hard) + device = scores.device + target_index = to_device(torch.tensor(targets), device) + ranks = None + if 'sort' in policies or records: + if self._device != device: + self._negative_template = torch.ones(self.n, dtype=torch.long, device=device) + self._outside_index = torch.tensor(sorted(self.outside), dtype=torch.long, device=device) + self._negative_template[self._outside_index] = 0 + self._positions = torch.arange(self.n, device=device) + self._device = device + # Sort before filtering, including outside-domain -inf entries, to + # preserve the reference evaluator's tie ordering exactly. + values = scores if scores.is_contiguous() else scores.clone() + if not self.full_domain: + values = scores.clone() + values[self._outside_index] = -torch.inf + order = values.argsort(descending=True) + negatives = self._negative_template.clone() + negatives.index_fill_(0, to_device(torch.tensor(sorted(answers), dtype=torch.long), device), 0) + positions = torch.empty_like(order) + positions[order] = self._positions + ranks = 1 + negatives[order].cumsum(0)[positions[target_index]] + if set(policies) != {'sort'} or records: + excluded = sorted(self.outside | answers) + eligible = torch.ones(self.n, dtype=torch.bool, device=device) + eligible.index_fill_(0, to_device(torch.tensor(excluded, dtype=torch.long), device), False) + # Eligible scores in ascending order; valid scores sort before +inf. + count = self.n - len(excluded) + negatives = torch.where(eligible, scores, torch.inf).sort().values[:count] + values = scores[target_index].contiguous() + lower = torch.searchsorted(negatives, values, right=False) + upper = torch.searchsorted(negatives, values, right=True) + starts, tied = count - upper + 1, upper - lower + packed = [invalid.double()[None]] + for policy in policies: + if policy == 'expected': + # Each position in the exact tie block is equally likely. + sizes = tied + 1 + last = starts + sizes - 1 + if self._harmonic is None or self._harmonic.device != scores.device: + self._harmonic = torch.cat((torch.zeros(1, dtype=torch.float64, device=scores.device), + torch.arange(1, self.n + 1, dtype=torch.float64, device=scores.device).reciprocal().cumsum(0))) + rr = (self._harmonic[last] - self._harmonic[starts - 1]) / sizes + hits = [((k - starts + 1).clamp_min(0).minimum(sizes).double() / sizes).mean() for k in (1, 3, 10)] + metrics = torch.stack([rr.mean(), *hits]) + else: + policy_ranks = (ranks.double() if policy == 'sort' else starts.double() + + {'average': .5, 'optimistic': 0., 'pessimistic': 1.}[policy] * tied) + metrics = torch.stack([policy_ranks.reciprocal().mean(), *((policy_ranks <= k).double().mean() for k in (1, 3, 10))]) + packed.append(metrics) + if records: + # Integer ranks are exact in float64 for any feasible vocabulary. + packed.append(torch.stack([target_index, ranks, starts - 1, tied + 1], dim=1).double().flatten()) + invalid, *host = torch.cat(packed).tolist() + if invalid: + raise ValueError('Expected one complete higher-is-better score vector') + result = {policy: dict(zip(METRICS, host[4 * i:4 * i + 4])) for i, policy in enumerate(policies)} + values = [int(value) for value in host[4 * len(policies):]] + detail = [values[i:i + 4] for i in range(0, len(values), 4)] if records else None + return result, detail + + +def filtered_query_metrics(scores, easy, hard, *, candidates=None, tie_policy='sort'): + """Filter other true answers and average over hard answers for one query. + + ``sort`` preserves PyTorch's descending argsort tie ordering, as in the + reference evaluator. Reuse QueryMetrics when evaluating a whole dataset. + """ + if scores.ndim != 1: + raise ValueError('Expected one complete higher-is-better score vector') + return QueryMetrics(len(scores), candidates=candidates, tie_policy=tie_policy)(scores, easy, hard) + + +def _averages(per_shape): + result = {} + for name, shapes in (('all', list(per_shape)), ('epfo', [s for s in per_shape if 'n' not in s]), + ('negation', [s for s in per_shape if 'n' in s])): + result[name] = ({metric: sum(per_shape[s][metric] for s in shapes) / len(shapes) for metric in METRICS} + | {'shapes': len(shapes)}) if shapes else None + return result + + +def _query_plan(data, limit, order, *, sampling='prefix', seed=0): + if limit is not None and (type(limit) is not int or limit < 1): + raise ValueError('Query limit must be a positive integer') + if order not in ('published', 'relation'): + raise ValueError('Unknown query order') + if sampling not in ('prefix', 'uniform') or type(seed) is not int: + raise ValueError('Choose prefix/uniform sampling and an integer sampling seed') + counts, selected = Counter(), [] + if sampling == 'uniform' and limit is not None: + groups = defaultdict(list) + for query in data.queries: + groups[query.shape].append(query) + for shape, queries in groups.items(): + def priority(query): + return fingerprint(('query-sample-v1', seed, data.name, data.split, shape, query.query)), query.query + selected.extend(heapq.nsmallest(limit, queries, key=priority)) + selected.sort(key=lambda q: (q.shape, q.query)) + else: + for query in data.queries: + if limit is None or counts[query.shape] < limit: + selected.append(query) + counts[query.shape] += 1 + if order == 'relation': + selected.sort(key=lambda q: (q.shape, relation_signature(compile_query(q.query)), q.query)) + return selected + + +def _plan_fingerprint(queries): + digest = hashlib.sha256() + for query in queries: + digest.update(query.identity.encode()) + return digest.hexdigest() + + +@torch.no_grad() +def evaluate_benchmark(data: QueryBenchmark, predict: Callable[[tuple], torch.Tensor], *, tie_policy: str = 'sort', + max_queries_per_shape: int | None = None, progress: Callable[[int, int], None] | None = None, + prepare: Callable[[list[tuple]], None] | None = None, query_batch_size: int = 1, + query_order: str = 'published', checkpoint_dir: str | Path | None = None, + checkpoint_identity: Any = None, checkpoint_every: int = 500, + on_checkpoint: Callable[[dict[str, Any]], None] | None = None, statistics: Counter | None = None, + query_sampling: str = 'prefix', sampling_seed: int = 0, + on_query: Callable[[tuple, str, dict[str, float]], None] | None = None, + additional_tie_policies: Sequence[str] = (), query_plan: Sequence[BenchmarkQuery] | None = None, + batch_ends: Sequence[int] | None = None, rank_trace_path: str | Path | None = None, + comparison_predictors: Mapping[str, Callable[[tuple], torch.Tensor]] | None = None, + filter_corrections: Mapping[tuple, frozenset | set] | None = None) -> dict[str, Any]: + """Stream filtered ranking metrics over a benchmark, resuming a verified query prefix. + + Each query's scores over all entities rank its hard answers after filtering + every other easy and hard answer. Metrics average answers per query, queries + per query type, and query types equally. + + Args: + data: Benchmark queries, inference graph and candidates. + predict: Scores of every entity for one query. + tie_policy: ``'sort'`` (PyTorch's default descending argsort, as in the + reference evaluator; not guaranteed stable, so exact ties may order + differently across devices) or ``'expected'`` (exact expectation + under random order within ties, which removes that dependence). + max_queries_per_shape: Sample at most this many queries per type. + progress: Called with completed and total query counts. + prepare: Called with each batch of queries before they are predicted. + query_batch_size: Queries per prepared batch. + query_order: Order of the query plan: ``'published'``, ``'canonical'`` or ``'relation'``. + checkpoint_dir: Directory for resumable progress. + checkpoint_identity: Everything that determines predictions; resuming requires equality. + checkpoint_every: Queries between saved checkpoints. + on_checkpoint: Called with the partial report at each checkpoint. + statistics: Counter of inference statistics to update and report. + query_sampling: ``'prefix'`` or ``'uniform'`` sampling of limited query plans. + sampling_seed: Seed of uniform sampling. + on_query: Called with each query, its type and its metrics. + additional_tie_policies: Further tie policies computed from the same scores. + query_plan: Explicit ordered queries instead of a sampled plan. + batch_ends: Explicit batch boundaries in ``query_plan``. + rank_trace_path: SQLite file recording every hard answer's ranks. + comparison_predictors: Named predictors scored on the same queries, such + as identity-adapter controls. + filter_corrections: Corrected easy answers by query; released filters are + then reported as a control. + + Returns: + The report: per-type and averaged metrics, coverage, protocol and timings. + + Raises: + ValueError: For inconsistent settings, or a checkpoint of different inputs. + """ + if type(query_batch_size) is not int or query_batch_size < 1 or type(checkpoint_every) is not int or checkpoint_every < 1: + raise ValueError('Batch and checkpoint intervals must be positive integers') + if query_plan is not None: + if max_queries_per_shape is not None: + raise ValueError('An explicit query plan cannot also specify a sampling limit') + selected = list(query_plan) + if not selected or len(set(selected)) != len(selected) or not set(selected) <= set(data.queries): + raise ValueError('Explicit query plan must contain distinct benchmark queries with unchanged answers') + else: + selected = _query_plan(data, max_queries_per_shape, query_order, sampling=query_sampling, seed=sampling_seed) + if batch_ends is not None: + batch_ends = list(batch_ends) + if (not batch_ends or any(type(end) is not int for end in batch_ends) + or batch_ends != sorted(set(batch_ends)) or batch_ends[0] < 1 or batch_ends[-1] != len(selected)): + raise ValueError('Explicit batch boundaries must cover the entire query plan') + if rank_trace_path is not None and checkpoint_dir is None: + raise ValueError('Rank traces require a resumable checkpoint directory') + comparisons: dict[str, Any] = dict(comparison_predictors or {}) + if any(not isinstance(name, str) or not name or name in ('.', '..') or Path(name).name != name + or not callable(predictor) for name, predictor in comparisons.items()): + raise ValueError('Comparisons require named predictors with safe directory names') + predictors: dict[str | None, Any] = {None: predict, **comparisons} + controls = {} + if filter_corrections is not None: + known = {query.query: query for query in data.queries} + candidate_set = set(data.candidates) + if set(filter_corrections) - known.keys(): + raise ValueError('Filter corrections reference unknown queries') + for query, easy in filter_corrections.items(): + if set(easy) & known[query].hard or not set(easy) <= candidate_set: + raise ValueError('Corrected filters must be valid non-target candidates') + controls = {name: (name + '-' if name else '') + 'released-filters' for name in predictors} + if set(controls.values()) & comparisons.keys(): + raise ValueError('Released-filter comparison names collide with predictors') + comparisons.update(dict.fromkeys(controls.values())) + metrics_for_query = QueryMetrics(data.context.num_entities, candidates=data.candidates, tie_policy=tie_policy) + if len(set(additional_tie_policies)) != len(additional_tie_policies) or tie_policy in additional_tie_policies: + raise ValueError('Tie policies must be distinct') + additional = {policy: QueryMetrics(data.context.num_entities, candidates=data.candidates, tie_policy=policy) + for policy in additional_tie_policies} + if checkpoint_dir is not None and checkpoint_identity is None: + raise ValueError('Checkpointing requires an explicit predictor identity') + identity = dict(predictor=checkpoint_identity, implementation=implementation_fingerprint(), + dataset=data.name, split=data.split, context=data.context.identity, metadata=data.metadata, + candidates=fingerprint(data.candidates), plan=_plan_fingerprint(selected), + tie_policy=tie_policy, limit=max_queries_per_shape, order=query_order, + additional_tie_policies=list(additional_tie_policies), + sampling=query_sampling, sampling_seed=sampling_seed, + batch=query_batch_size, checkpoint_every=checkpoint_every) if checkpoint_dir is not None else None + if identity is not None: + identity.update(explicit_batches=batch_ends, rank_trace=rank_trace_path is not None) + if comparisons: + identity['comparisons'] = list(comparisons) + if filter_corrections is not None: + identity['filter_corrections'] = fingerprint(sorted((fingerprint(query), sorted(easy)) for query, easy in filter_corrections.items())) + store = BenchmarkCheckpoint(checkpoint_dir, identity) if checkpoint_dir is not None else nullcontext(None) + available = Counter(q.shape for q in data.queries) + stats = Counter() if statistics is None else statistics + with ExitStack() as stack: + checkpoint = stack.enter_context(store) + saved = checkpoint.load() if checkpoint else None + counts, hard_counts = Counter(), Counter() + completed, elapsed = 0, 0. + policies = (tie_policy, *additional) + # Metric sums per (predictor, tie policy) and query type; predictor None is ``predict``. + totals = {(name, policy): defaultdict(Counter) for name in (None, *comparisons) for policy in policies} + if saved is not None: + completed, elapsed = saved['completed'], saved['seconds'] + if not 0 <= completed <= len(selected): + raise ValueError('Invalid checkpoint query count') + counts.update(saved['counts']) + hard_counts.update(saved['hard_counts']) + for name, policy, values in saved['totals']: + totals[name, policy].update({shape: Counter(sums) for shape, sums in values.items()}) + stats.update({key: value for key, value in saved['statistics'].items() + if key not in ('cache_bytes', 'cache_peak_bytes')}) + stats['cache_peak_bytes'] = max(stats.get('cache_peak_bytes', 0), + saved['statistics'].get('cache_peak_bytes', 0)) + if completed == len(selected): + # A completed checkpoint replays the last evaluation's report. + stats['cache_bytes'] = saved['statistics'].get('cache_bytes', 0) + else: + # Continuing inference starts with fresh process-local caches. + stats.setdefault('cache_bytes', 0) + if sum(counts.values()) != completed: + raise ValueError('Invalid checkpoint shape counts') + if batch_ends is not None and completed and completed not in batch_ends: + raise ValueError('Checkpoint cuts through a reference batch') + trace, comparison_traces = None, {} + if rank_trace_path is not None and identity is not None: + trace = stack.enter_context(RankTrace(rank_trace_path, identity, completed)) + comparison_traces = {name: stack.enter_context(RankTrace(Path(rank_trace_path).parent / name / 'ranks.sqlite3', + dict(identity, comparison=name), completed)) + for name in comparisons} + timings = Counter(saved.get('timings', {}) if saved else {}) + started = time.monotonic() + + def report(variant: str | None = None) -> dict[str, Any]: + def means(policy): + return {shape: {metric: totals[variant, policy][shape][metric] / counts[shape] for metric in METRICS} + | dict(queries=counts[shape], hard_answers=hard_counts[shape], available_queries=available[shape]) + for shape in sorted(counts)} + + per_shape, extra = means(tie_policy), {} + for policy in additional: + values = means(policy) + extra[policy] = dict(per_shape=values, averages=_averages(values)) + result = dict(dataset=data.name, group=data.group, split=data.split, per_shape=per_shape, averages=_averages(per_shape), + queries=completed, seconds=elapsed + time.monotonic() - started, + protocol=dict(tie_policy=tie_policy, filtering='all other easy and hard answers', + averaging='answers per query, queries per shape, equal shapes per group', + max_queries_per_shape=max_queries_per_shape, query_order=query_order, + query_sampling=query_sampling, sampling_seed=sampling_seed, + full_split=completed == len(data.queries), + all_14_shapes=set(counts) == set(ULTRAQUERY_SHAPES), + all_benchmark_shapes=set(counts) == set(data.metadata.get('expected_query_types', ULTRAQUERY_SHAPES))), + dataset_metadata=data.metadata, context_sha256=data.context.identity, + candidate_sha256=fingerprint(data.candidates), num_candidates=len(data.candidates), + timings=dict(timings), + **({'additional_tie_metrics': extra} if extra else {})) + if filter_corrections is not None: + released = variant in controls.values() + result['protocol']['answer_filter'] = 'released' if released else 'corrected' + result['dataset_metadata'] = dict(data.metadata, answer_filter=result['protocol']['answer_filter']) + if released: + result['filter_control_of'] = next(name or '' for name, control in controls.items() if control == variant) + if variant is None and comparisons: + result['comparisons'] = {name: report(name) for name in comparisons} + return result + + if progress and completed: + progress(completed, len(selected)) + while completed < len(selected): + end = (batch_ends[bisect_right(batch_ends, completed)] if batch_ends is not None else + min(completed + query_batch_size, len(selected), (completed // checkpoint_every + 1) * checkpoint_every)) + batch = selected[completed:end] + step_started = time.monotonic() + if prepare: + prepare([q.query for q in batch]) + timings['prepare_wall_seconds'] += time.monotonic() - step_started + checkpoint_due = False + for q in batch: + for name, predictor in predictors.items(): + step_started = time.monotonic() + scores = predictor(q.query) + timings['predict_dispatch_wall_seconds'] += time.monotonic() - step_started + if scores.shape != (data.context.num_entities,): + raise ValueError('Predictor must score the complete public entity vocabulary') + step_started = time.monotonic() + easy = q.easy if filter_corrections is None else filter_corrections.get(q.query, q.easy) + all_metrics, details = metrics_for_query.evaluate(scores, easy, q.hard, (tie_policy, *additional), records=trace is not None) + main_metrics = all_metrics[tie_policy] + variants = [(name, all_metrics, details)] + if controls: + released = ((all_metrics, details) if easy == q.easy else + metrics_for_query.evaluate(scores, q.easy, q.hard, (tie_policy, *additional), records=trace is not None)) + variants.append((controls[name], *released)) + for variant, values, records in variants: + for policy in policies: + totals[variant, policy][q.shape].update(values[policy]) + current_trace = trace if variant is None else comparison_traces.get(variant) + if current_trace is not None: + current_trace.add(completed, q, records) + timings['metrics_and_trace_wall_seconds'] += time.monotonic() - step_started + if on_query and name is None: + on_query(q.query, q.shape, dict(main_metrics)) + counts[q.shape] += 1 + hard_counts[q.shape] += len(q.hard) + completed += 1 + checkpoint_due |= completed % checkpoint_every == 0 or completed == len(selected) + if progress and completed < end: + progress(completed, len(selected)) + if checkpoint_due: + current = report() + if trace is not None: + trace.commit() + for comparison_trace in comparison_traces.values(): + comparison_trace.commit() + if checkpoint: + checkpoint.save(dict(completed=completed, seconds=current['seconds'], counts=dict(counts), + hard_counts=dict(hard_counts), statistics=dict(stats), timings=dict(timings), + totals=[[name, policy, dict(values)] for (name, policy), values in totals.items()])) + if on_checkpoint: + on_checkpoint(current) + if progress: + progress(completed, len(selected)) + return report() + + +def benchmark_model(model: torch.nn.Module, data: QueryBenchmark, *, adapter: QueryScoreAdapter | None = None, + observed_mix: float | None = None, beam_size: int = 64, tnorm: str = 'prod', row_batch_size: int = 8, + backend_batch_size: int | None = None, cache_bytes: int = 512 * 1024 * 1024, seed: int = 0, + samples: int | None = None, tie_policy: str = 'sort', max_queries_per_shape: int | None = None, + progress: Callable[[int, int], None] | None = None, query_batch_size: int = 32, + query_order: str = 'relation', checkpoint_dir: str | Path | None = None, checkpoint_every: int = 500, + on_checkpoint: Callable[[dict[str, Any]], None] | None = None, query_sampling: str = 'prefix', + sampling_seed: int = 0, on_query: Callable[[tuple, str, dict[str, float]], None] | None = None, + executor: str = 'cqd') -> dict[str, Any]: + """Evaluate a KGE or KGFM with CQD beam search, shared atomic caches and durable progress. + + Arguments not listed here are those of ``evaluate_benchmark``. + + Args: + model: Link predictor scoring ``(head, relation)`` rows over all entities. + data: Benchmark to evaluate. + adapter: Score adapter; None uses sigmoid scores. + observed_mix: Overrides the adapter's weight of observed facts. + beam_size: Intermediate candidates kept per variable. + tnorm: ``'prod'`` or ``'min'``. + row_batch_size: Atomic rows scored per backbone call. + backend_batch_size: Query batch of graph backbones. + cache_bytes: Budget of the calibrated-row cache. + seed: Seed of sampled scoring. + samples: Number of sampled scoring passes, if any. + executor: Query executor, ``'cqd'``. + + Returns: + The ``evaluate_benchmark`` report with inference settings and cache statistics. + """ + from ..models._inference import float32_precision_token + from ..models.flock import Flock + if executor not in ('cqd', 'qto'): + raise ValueError('Choose cqd or qto executor') + previous_batch = getattr(model, 'query_batch_size', None) + effective_batch = backend_batch_size + if effective_batch is None: + effective_batch = previous_batch if isinstance(model, Flock) else row_batch_size + if type(effective_batch) is not int or effective_batch < 1: + raise ValueError('Backend batch size must be a positive integer') + stats = Counter() + try: + if previous_batch is not None: + setattr(model, 'query_batch_size', effective_batch) + with attached_context(model, data.context), evaluation_mode(model): + engine = QueryAnswerer(model, context=data.context, adapter=adapter, observed_mix=observed_mix, + row_batch_size=row_batch_size, cache_bytes=cache_bytes, seed=seed, samples=samples) + transform = engine.adapter + inference = dict(method='cqd-global-prefix' if executor == 'cqd' else 'qto-exact', executor=executor, + model=model.name, beam_size=beam_size if executor == 'cqd' else None, tnorm=tnorm, + negation='standard', score_space='float64-log-memberships', + row_batch_size=row_batch_size, backend_batch_size=effective_batch if previous_batch is not None else None, + query_batch_size=query_batch_size, cache_bytes=cache_bytes, cache_scope='evaluation-session', + seed=seed, samples=samples, + effective_samples=getattr(model, 'test_samples', None) if samples is None else samples, + sampling='independent-row-single-sample-v1' if isinstance(model, Flock) else None, + adapter=transform.to_dict(), + backbone_state_sha256=state_fingerprint(model), device=str(engine.scorer.device), + cuda=torch.version.cuda, + gpu=torch.cuda.get_device_name(engine.scorer.device) if engine.scorer.device.type == 'cuda' else None, + torch=str(torch.__version__), float32_precision=str(float32_precision_token()), + cpu_threads=torch.get_num_threads(), + deterministic=torch.are_deterministic_algorithms_enabled(), + model_configuration={k: str(v) for k, v in getattr(model, 'args', {}).items()}, + runtime_configuration={k: getattr(model, k, None) for k in ( + '_inference_backend', 'query_batch_size', 'relation_cache_mb', 'projection_cache_mb', + 'walk_num', 'walk_len', 'refinements', 'compact_state', 'pack_walks', 'compile_sampler', 'prefetch_walks')}, + module_configuration={name: {k: getattr(module, k, None) for k in ('inference_backend', 'inference_compile')} + for name, module in model.named_modules() if hasattr(module, 'inference_backend')}, + implementation_sha256=implementation_fingerprint()) + + def predict(query): + result = engine.predict(query, beam_size=beam_size, tnorm=tnorm, return_log_scores=True, executor=executor) + _accumulate_inference_statistics(stats, engine.last_info, (engine,)) + return result + + def prepare(queries): + _accumulate_inference_statistics(stats, engine.prefetch(queries), (engine,)) + + def annotate(report: dict[str, Any]) -> dict[str, Any]: + return dict(report, inference=dict(inference, statistics=dict(stats))) + + report = evaluate_benchmark(data, predict, tie_policy=tie_policy, + max_queries_per_shape=max_queries_per_shape, progress=progress, + prepare=prepare, query_batch_size=query_batch_size, query_order=query_order, + query_sampling=query_sampling, sampling_seed=sampling_seed, on_query=on_query, + checkpoint_dir=checkpoint_dir, checkpoint_identity=inference, checkpoint_every=checkpoint_every, + on_checkpoint=(lambda report: on_checkpoint(annotate(report))) if on_checkpoint else None, + statistics=stats) + return annotate(report) + finally: + if previous_batch is not None: + setattr(model, 'query_batch_size', previous_batch) + + +def summarize_benchmarks(reports): + """Equal dataset averages; partial selections are labeled, never padded with zeros.""" + if len({(r['dataset'], r['split']) for r in reports}) != len(reports): + raise ValueError('Duplicate dataset/split results cannot be averaged') + if len({r['split'] for r in reports}) > 1: + raise ValueError('Do not aggregate validation and test splits together') + if len({r['dataset_metadata'].get('suite', 'ultraquery') for r in reports}) > 1: + raise ValueError('Do not aggregate UltraQuery and +H benchmark suites together') + if len({r['dataset_metadata'].get('evaluation', 'hard-answers') for r in reports}) > 1: + raise ValueError('Do not aggregate faithfulness and hard-answer evaluations together') + result = {} + for group in ('all', 'transductive', 'inductive-e', 'inductive-er'): + selected = [r for r in reports if group == 'all' or r['group'] == group] + if not selected: + continue + result[group] = {'datasets': len(selected)} + for category in ('all', 'epfo', 'negation'): + values = [r['averages'][category] for r in selected if r['averages'][category] is not None] + result[group][category] = ({metric: sum(v[metric] for v in values) / len(values) for metric in METRICS} + | {'datasets': len(values)}) if values else None + return dict(groups=result, complete_23_dataset_test_suite=(len(reports) == 23 and + {r['dataset'] for r in reports} == set(BENCHMARK_DATASETS) and + all(r['split'] == 'test' and r['protocol']['full_split'] and r['protocol']['all_14_shapes'] for r in reports)), + complete_plus_h_test_suite=({r['dataset'] for r in reports} == set(PLUS_H_DATASETS) and + all(r['split'] == 'test' and r['protocol']['full_split'] and r['protocol'].get('all_benchmark_shapes', False) + for r in reports))) + + +def add_benchmark_parser(commands): + parser = commands.add_parser('benchmark', help='Evaluate CQD/QTO on published UltraQuery or +H datasets') + parser.add_argument('--data-root', type=Path, required=True) + parser.add_argument('--datasets', nargs='+', default=['FB15k237LogicalQuery'], help='Dataset names, all for UltraQuery, or +h for the three harder datasets') + parser.add_argument('--split', choices=['valid', 'test'], default='test') + parser.add_argument('--query-types', nargs='+', choices=list(QUERY_SHAPES)) + parser.add_argument('--download', action='store_true', help='Download official archives if missing') + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument('--experiment', help='Saved DICE experiment') + source.add_argument('--checkpoint', type=Path, help='Pretrained graph-model checkpoint') + parser.add_argument('--model', choices=['ULTRA', 'TRIX', 'Flock', 'KGICL'], default='ULTRA') + parser.add_argument('--model-config', type=Path, help='Optional model constructor arguments as JSON') + parser.add_argument('--adapter', type=Path, help='Fitted DICE adapter JSON') + parser.add_argument('--observed-mix', type=float) + parser.add_argument('--beam-size', type=int, default=64) + parser.add_argument('--executor', choices=['cqd', 'qto'], default='cqd', help='Beam search or exact QTO-style projection') + parser.add_argument('--tnorm', choices=['prod', 'min'], default='prod') + parser.add_argument('--row-batch-size', type=int, default=8) + parser.add_argument('--backend-batch-size', type=int, help='Neural microbatch; defaults to row batch size for ULTRA/TRIX') + parser.add_argument('--query-batch-size', type=int, default=32, help='Queries whose anchor rows are prefetched together') + parser.add_argument('--query-order', choices=['published', 'relation'], default='relation') + parser.add_argument('--cache-mb', type=int, default=512) + parser.add_argument('--checkpoint-every', type=int, default=500) + parser.add_argument('--tie-policy', choices=['sort', 'average', 'optimistic', 'pessimistic'], default='sort') + parser.add_argument('--max-queries-per-shape', type=int, help='Deterministic smoke-test subset; omitted means full evaluation') + parser.add_argument('--query-sampling', choices=['prefix', 'uniform'], default='prefix', help='Uniform sampling is nested across query limits') + parser.add_argument('--sampling-seed', type=int, default=0, help='Query selection seed, independent of backbone sampling') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--samples', type=int) + parser.add_argument('--device', default='cpu') + parser.add_argument('--threads', type=int, default=2) + parser.add_argument('--precision', choices=['ieee', 'tf32'], default='ieee') + parser.add_argument('--output', type=Path, required=True, help='JSON report with automatic checkpoint/resume') + + +def run_benchmark_cli(args): + from ..models import KGICL, TRIX, ULTRA, Flock + from ..models._inference import float32_precision_backends + from ..models.graph_model import GraphKGE + if args.threads < 1: + raise ValueError('Thread count must be positive') + torch.set_num_threads(args.threads) + backends = float32_precision_backends() + if backends: + for backend in backends: + backend.fp32_precision = args.precision + else: + torch.set_float32_matmul_precision('highest' if args.precision == 'ieee' else 'high') + torch.backends.cudnn.allow_tf32 = args.precision == 'tf32' + names = (list(BENCHMARK_DATASETS) if args.datasets == ['all'] else + list(PLUS_H_DATASETS) if args.datasets in (['+h'], ['+H']) else args.datasets) + if len(set(names)) != len(names): + raise ValueError('Choose each dataset once') + if any(name in PLUS_H_DATASETS for name in names) and not all(name in PLUS_H_DATASETS for name in names): + raise ValueError('Run UltraQuery and +H benchmark suites with separate output paths') + kge = None + if args.experiment: + from ..knowledge_graph_embeddings import KGE + kge = KGE(path=args.experiment) + model = kge.model + else: + configuration = json.loads(args.model_config.read_text()) if args.model_config else {} + model = {'ULTRA': ULTRA, 'TRIX': TRIX, 'Flock': Flock, 'KGICL': KGICL}[args.model](dict(configuration, num_entities=1, num_relations=1)) + model.load_pretrained(args.checkpoint) + model.to(args.device) + adapter = QueryScoreAdapter.load(args.adapter, model=model) if args.adapter else None + request = dict(arguments={key: str(value) for key, value in vars(args).items()}, + implementation=implementation_fingerprint(), backbone=state_fingerprint(model), + adapter=json.loads(args.adapter.read_text()) if args.adapter else None) + directory = args.output.with_suffix(args.output.suffix + '.checkpoints') + with BenchmarkCheckpoint(directory, request): + try: + _run_datasets(args, names, model, adapter, kge, directory, GraphKGE) + except BaseException as error: + if args.output.exists(): + payload = json.loads(args.output.read_text()) + write_json(args.output, dict(payload, state='failed', error=str(error))) + raise + + +def _run_datasets(args, names, model, adapter, kge, directory, graph_type): + reports = [] + args.output.parent.mkdir(parents=True, exist_ok=True) + for name in names: + print(f'Loading {name} ({args.split})', flush=True) + data = load_benchmark(args.data_root, name, split=args.split, query_types=args.query_types, download=args.download) + if not isinstance(model, graph_type): + if data.entity_to_idx is None or kge.entity_to_idx != data.entity_to_idx or kge.relation_to_idx != data.relation_to_idx: + raise ValueError('Transductive experiment IDs must exactly match the published benchmark vocabulary') + + def progress(done, total): + if done == 1 or done % 100 == 0 or done == total: + print(f'{name}: {done}/{total} queries', flush=True) + + def checkpoint(report): + write_json(args.output, dict(version=2, state='running', results=reports, current_result=report, + summary=summarize_benchmarks(reports))) + + report = benchmark_model(model, data, adapter=adapter, observed_mix=args.observed_mix, + beam_size=args.beam_size, executor=args.executor, tnorm=args.tnorm, row_batch_size=args.row_batch_size, + backend_batch_size=args.backend_batch_size, query_batch_size=args.query_batch_size, + query_order=args.query_order, checkpoint_dir=directory / name.replace(':', '-'), + query_sampling=args.query_sampling, sampling_seed=args.sampling_seed, + checkpoint_every=args.checkpoint_every, on_checkpoint=checkpoint, + cache_bytes=args.cache_mb * 2**20, seed=args.seed, samples=args.samples, + tie_policy=args.tie_policy, max_queries_per_shape=args.max_queries_per_shape, progress=progress) + reports.append(report) + write_json(args.output, dict(version=2, state='complete' if len(reports) == len(names) else 'running', + results=reports, current_result=None, summary=summarize_benchmarks(reports))) + print(f'{name}: {json.dumps(report["averages"])}', flush=True) + print(f'Saved {args.output}', flush=True) diff --git a/dicee/query_answering/catalog.py b/dicee/query_answering/catalog.py new file mode 100644 index 000000000..fff0a5f18 --- /dev/null +++ b/dicee/query_answering/catalog.py @@ -0,0 +1,164 @@ +"""Published complex-query benchmarks, the upstream code they reproduce, and recipe rules. + +The benchmark launcher loads this file without PyTorch, so it imports nothing. +""" + +# Query structures in the released grammar: 'e' anchor, 'r' relation, 'n' negation, 'u' union. +_ONE = ('e', ('r',)) +QUERY_SHAPES = { + '1p': _ONE, '2p': ('e', ('r', 'r')), '3p': ('e', ('r', 'r', 'r')), + '2i': (_ONE, _ONE), '3i': (_ONE, _ONE, _ONE), + 'ip': ((_ONE, _ONE), ('r',)), 'pi': (('e', ('r', 'r')), _ONE), + '2u': (_ONE, _ONE, ('u',)), 'up': ((_ONE, _ONE, ('u',)), ('r',)), + '2in': (_ONE, ('e', ('r', 'n'))), '3in': (_ONE, _ONE, ('e', ('r', 'n'))), + 'inp': ((_ONE, ('e', ('r', 'n'))), ('r',)), + 'pin': (('e', ('r', 'r')), ('e', ('r', 'n'))), + 'pni': (('e', ('r', 'r', 'n')), _ONE), + '4p': ('e', ('r', 'r', 'r', 'r')), '4i': (_ONE, _ONE, _ONE, _ONE), +} +ULTRAQUERY_SHAPES = ('1p', '2p', '3p', '2i', '3i', 'ip', 'pi', '2u', 'up', '2in', '3in', 'inp', 'pin', 'pni') +PLUS_H_SHAPES = (*ULTRAQUERY_SHAPES, '4p', '4i') + +# Released datasets of the UltraQuery suite, by public name. +TRANSDUCTIVE = {'FB15k237LogicalQuery': 'FB15k-237-betae', 'FB15kLogicalQuery': 'FB15k-betae', 'NELL995LogicalQuery': 'NELL-betae'} +INDUCTIVE_VERSIONS = ('106', '113', '122', '134', '150', '175', '217', '300', '550') +WIKITOPICS = ('art', 'award', 'edu', 'health', 'infra', 'loc', 'org', 'people', 'sci', 'sport', 'tax') +BENCHMARK_DATASETS = (*TRANSDUCTIVE, *(f'InductiveFB15k237Query:{v}' for v in INDUCTIVE_VERSIONS), + *(f'WikiTopicsQuery:{v}' for v in WIKITOPICS)) + +# The harder +H suite and its single release archive. +PLUS_H_FOLDERS = {'FB15k237+H': 'FB15k-237+H', 'NELL995+H': 'NELL995+H', 'ICEWS18+H': 'ICEWS18+H'} +PLUS_H_DATASETS = tuple(PLUS_H_FOLDERS) +PLUS_H_ARCHIVE = 'https://github.com/april-tools/is-cqa-complex/releases/download/benchs-1.0/iscqa-compl-benchmarks.zip' +PLUS_H_PREFIX = 'iscqa-compl-benchmarks/new_benchmarks' + +# Pinned upstream implementations: method -> (repository, commit). +ULTRA = ('DeepGraphLearning/ULTRA', '427966ad8ed60420eef034063d44f3153addff90') +TRIX = ('yuchengz99/TRIX', '7596e14eefefe89e61396205a0550172cadeddb0') +KG_ICL = ('nju-websoft/KG-ICL', '6a3166e347ae468acdfb30a70a2cf3608b66b8f1') +FLOCK = ('jw9730/flock', 'f35103d25a78bdf4075de5c673a51de4979aa4d7') +INDUCTIVE_QE = ('DeepGraphLearning/InductiveQE', 'ab49a0e64e3449e687e484a42269d9898e1d1325') +IS_CQA = ('april-tools/is-cqa-complex', 'd1ce74164936a7c09d9147e83190da047cb39429') +REFERENCES = { + 'ultraquery': ULTRA, + 'ultraquery-lp': ULTRA, + 'inductive-gnnqe': INDUCTIVE_QE, + 'incoming-relation': INDUCTIVE_QE, + 'gnnqe': ('DeepGraphLearning/GNN-QE', '6620fe3d8d77ceed019f36541e84d67599d4f259'), + 'cqd': IS_CQA, + 'cqd-hybrid': IS_CQA, + 'qto': ('bys0318/QTO', '31f46b970d0f075e6b2be9c733cc11b41365d5fe'), + 'cone': ('MIRALab-USTC/QE-ConE', 'cc45b90e3cdce0f609670a257e6f756d28495a93'), + 'clmpt': ('qianlima-lab/CLMPT', '6c4f3b8a5e052e8bc1c0c93eaed18ef869a1bcd7'), +} +KGFM_ADAPTERS = ('ultra-adapter', 'trix-adapter', 'kgicl-adapter', 'flock-adapter') +# The upstream backbone each adapter method reproduces (KG-ICL with docs/kgicl.md's bug fixes). +KGFM_BACKBONES = {'ultra-adapter': ULTRA, 'trix-adapter': TRIX, 'kgicl-adapter': KG_ICL, 'flock-adapter': FLOCK} +METHODS = (*REFERENCES, *KGFM_ADAPTERS) +# Methods whose checkpoints score queries without the inference graph. +GRAPH_INDEPENDENT_METHODS = ('cone', 'clmpt', 'cqd') + + +def dataset_spec(name: str) -> tuple[str, str, str]: + """Locate a released benchmark. + + Args: + name: A name from ``BENCHMARK_DATASETS`` or ``PLUS_H_DATASETS``. + + Returns: + The setting (``'transductive'``, ``'inductive-e'`` or ``'inductive-er'``), + the dataset folder below the data root, and the release archive URL. + + Raises: + ValueError: If ``name`` is not a released benchmark. + """ + if name in PLUS_H_FOLDERS: + return 'transductive', f'{PLUS_H_PREFIX}/{PLUS_H_FOLDERS[name]}', PLUS_H_ARCHIVE + if name in TRANSDUCTIVE: + return 'transductive', TRANSDUCTIVE[name], 'https://snap.stanford.edu/betae/KG_data.zip' + family, _, version = name.partition(':') + if family in ('InductiveFB15k237Query', 'InductiveFB15k237QueryExtendedEval') and version in INDUCTIVE_VERSIONS: + return 'inductive-e', version, f'https://zenodo.org/records/7306046/files/{version}.zip' + if family == 'WikiTopicsQuery' and version in WIKITOPICS: + return 'inductive-er', f'WikiTopics_QE/{version}', 'https://reltrans.s3.us-east-2.amazonaws.com/WikiTopics_QE.zip' + raise ValueError(f'Unknown benchmark {name!r}; choose from BENCHMARK_DATASETS or PLUS_H_DATASETS') + + +def query_types_for_dataset(name: str) -> tuple[str, ...]: + """The query types released for benchmark ``name``: 16 for +H, 14 otherwise.""" + dataset_spec(name) + return PLUS_H_SHAPES if name in PLUS_H_DATASETS else ULTRAQUERY_SHAPES + + +def inference_graph_for_split(name: str, split: str, inference_graph: str | None = None) -> str | None: + """The facts that answer +H queries of ``split``; None for datasets with released inference graphs. + + Args: + name: Benchmark name. + split: ``'valid'`` or ``'test'``. + inference_graph: ``'train'`` or ``'train+valid'`` for +H test queries + (default: train+valid). Validation always uses training facts. + + Raises: + ValueError: For an unknown graph, or an override outside +H. + """ + if inference_graph not in (None, 'train', 'train+valid'): + raise ValueError('Inference graph must be train or train+valid') + if name not in PLUS_H_DATASETS: + if inference_graph is not None: + raise ValueError('Inference graph overrides are only supported for +H') + return None + return (inference_graph or 'train+valid') if split == 'test' else 'train' + + +def check_recipe(entry: dict, shapes: list[str] | tuple[str, ...]) -> None: + """Check the method rules of an evaluation recipe. + + Shared by the evaluator and the benchmark manifests, so both reject the + same recipes before any data or checkpoint is opened. + + Args: + entry: Recipe with ``method`` and optional ``options``, ``query_batch_size`` + and, for KGFM adapters, ``operators``, ``adapters``, ``selection_protocol`` + and ``adapter_ablation``. + shapes: Query types the recipe evaluates. + + Raises: + ValueError: If the recipe breaks a rule of its method. + """ + method, options = entry['method'], entry.get('options', {}) + if method not in METHODS: + raise ValueError(f'Choose a supported query method: {", ".join(METHODS)}') + if method in ('cqd', 'cqd-hybrid') and any('n' in shape for shape in shapes) and not options.get('atomic_negation'): + raise ValueError('CQD negated query types require atomic_negation=true; otherwise select positive query types') + size = entry.get('query_batch_size', 1) + if type(size) is not int or size < 1: + raise ValueError('Query batch size must be positive') + if method not in KGFM_ADAPTERS: + return + if options.get('observed_facts', 'none') not in ('none', 'atomic', 'both'): + raise ValueError('Observed facts must be none, atomic, or both') + if options.get('observed_ties', 'entity') not in ('entity', 'model'): + raise ValueError('Observed ties must be entity or model') + if options.get('negation', 'model') not in ('model', 'observed'): + raise ValueError('Negation must be model or observed') + if 'prompt_seed' in options and (method != 'kgicl-adapter' or type(options['prompt_seed']) is not int + or not 0 <= options['prompt_seed'] < 2**63): + raise ValueError('Only KG-ICL recipes take prompt_seed, a nonnegative integer below 2**63') + if 'relation_conditioning' in options and (method != 'ultra-adapter' or options['relation_conditioning'] not in ('direct', 'query')): + raise ValueError('Only ULTRA recipes take relation_conditioning, direct or query') + if any(name in options and (method != 'flock-adapter' or type(options[name]) is not int or options[name] < 1) + for name in ('test_samples', 'walk_num')): + raise ValueError('Only Flock recipes take test_samples and walk_num, positive integers') + if type(entry.get('adapter_ablation', False)) is not bool: + raise ValueError('Adapter ablation must be a boolean') + if entry.get('selection_protocol') not in ('source-validation', 'target-validation'): + raise ValueError('Declare the operator selection protocol') + operators = entry.get('operators', {}) + + def valid(operator): # A t-norm, optionally labelled to route query types to different adapters ('product:chains'). + norm, colon, label = operator.partition(':') + return norm in ('product', 'min') and (not colon or (label != '' and all(c.isalnum() or c in '-_' for c in label))) + + if set(operators) != set(shapes) or not all(map(valid, operators.values())) or set(operators.values()) - entry.get('adapters', {}).keys(): + raise ValueError('Each query type needs one operator, product or min (optionally labelled), with its matching adapter') diff --git a/dicee/query_answering/context.py b/dicee/query_answering/context.py new file mode 100644 index 000000000..72806bdfc --- /dev/null +++ b/dicee/query_answering/context.py @@ -0,0 +1,176 @@ +"""Public-ID graph context; answers and evaluation labels never enter this object.""" + +import hashlib +import json +import math +from collections import Counter, defaultdict +from contextlib import contextmanager +from dataclasses import dataclass + +import torch + +from ..models._inference import to_device +from ._query import exact_answers + + +def fingerprint(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def state_fingerprint(model_or_state): + state = model_or_state.state_dict() if hasattr(model_or_state, 'state_dict') else model_or_state + digest = hashlib.sha256() + for name, tensor in sorted(state.items()): + tensor = tensor.detach().cpu().contiguous() + digest.update(json.dumps([name, str(tensor.dtype), list(tensor.shape)]).encode()) + digest.update(tensor.reshape(-1).view(torch.uint8).numpy().tobytes()) + return digest.hexdigest() + + +def checkpoint_state(path): + """Tensor state of a backbone checkpoint: bare, under 'model' (ULTRA, TRIX, Flock) or 'state_dict' (KG-ICL).""" + state = torch.load(path, map_location='cpu', weights_only=True) + if isinstance(state.get('state_dict'), dict): + return state['state_dict'] + return state.get('model', state) + + +def state_token(model): + """Cheap mutation token; inference tensors cannot safely be retained in caches. + + Use normal PyTorch parameter/buffer updates (copy_, load_state_dict, optimizers), + not writes through .data, which bypass PyTorch's mutation counters. + """ + tensors = (*model.named_parameters(), *model.named_buffers()) + if any(t.is_inference() for _, t in tensors): + return None + return (id(model), tuple((name, id(t), t._version, t.device, t.dtype, tuple(t.shape)) + for name, t in tensors)) + + +@dataclass(frozen=True) +class QueryContext: + """A complete public vocabulary and a deduplicated set of observed triples. + + Explicit inverse pairs use public direct/inverse IDs, never an assumed ID parity. + Supplied inverse pairs add reciprocal facts to the context. + """ + + triples: tuple + num_entities: int + num_relations: int + inverse_relations: tuple = () + + def __post_init__(self): + if type(self.num_entities) is not int or type(self.num_relations) is not int or min(self.num_entities, self.num_relations) < 1: + raise ValueError('Positive integer vocabulary sizes are required') + pairs = dict(self.inverse_relations) + ids = [r for pair in pairs.items() for r in pair] + if len(set(ids)) != len(ids) or any(type(r) is not int or not 0 <= r < self.num_relations for r in ids): + raise ValueError('Inverse pairs must be disjoint public relation IDs') + triples = self.triples.tolist() if hasattr(self.triples, 'tolist') else self.triples + edges = set() + inverse = {a: b for h, t in pairs.items() for a, b in ((h, t), (t, h))} + for triple in triples: + if len(triple) != 3 or any(type(v) is not int for v in triple): + raise ValueError('Context triples must contain three integer IDs') + h, r, t = triple + if not (0 <= h < self.num_entities and 0 <= t < self.num_entities and 0 <= r < self.num_relations): + raise ValueError('Context triple outside the public vocabulary') + edges.add((h, r, t)) + if r in inverse: + edges.add((t, inverse[r], h)) + object.__setattr__(self, 'triples', tuple(sorted(edges))) + object.__setattr__(self, 'inverse_relations', tuple(sorted(pairs.items()))) + out = defaultdict(lambda: defaultdict(set)) + for h, r, t in edges: + out[h][r].add(t) + object.__setattr__(self, 'outgoing', {h: dict(rs) for h, rs in out.items()}) + object.__setattr__(self, 'heads', Counter(h for h, _, _ in edges)) + object.__setattr__(self, 'relations', Counter(r for _, r, _ in edges)) + object.__setattr__(self, '_identity', fingerprint(self.to_dict())) + + @classmethod + def from_model(cls, model): + """Translate GraphKGE's internal reciprocal IDs back into its public IDs.""" + model._require_graph() + mapping = model.relation_id_map.detach().cpu().tolist() + public = {internal: external for external, internal in enumerate(mapping)} + direct_count = model.num_direct_relations + pairs = {external: public[internal + direct_count] for internal, external in public.items() + if internal < direct_count and internal + direct_count in public} + triples = [(h, public[r], t) for h, r, t in model.graph_triples.detach().cpu().tolist() if r in public] + return cls(triples, model.num_entities, model.num_relations, tuple(pairs.items())) + + @property + def identity(self): + return self._identity + + def to_dict(self): + return dict(triples=self.triples, num_entities=self.num_entities, num_relations=self.num_relations, + inverse_relations=self.inverse_relations) + + def answers(self, tree): + return exact_answers(tree, self.outgoing, self.num_entities) + + def features(self, conditions, *, device): + # Build on the host, then copy once: per-row device indexing waits for queued kernels. + observed = torch.zeros((len(conditions), self.num_entities), dtype=torch.bool) + rows, columns, base = [], [], [] + n, nr = self.num_entities, self.num_relations + for i, (h, r) in enumerate(conditions): + tails = self.outgoing.get(h, {}).get(r, ()) + rows.extend([i] * len(tails)) + columns.extend(tails) + base.append([1., math.log1p(len(tails)) / math.log1p(n), + math.log1p(self.heads[h]) / math.log1p(n * nr), + math.log1p(self.relations[r]) / math.log1p(n * n)]) + if columns: + observed[rows, columns] = True + return to_device(observed, device), to_device(torch.tensor(base, dtype=torch.float64), device) + + +@contextmanager +def attached_context(model, context): + """Attach an inference context temporarily and restore the caller's graph.""" + from ..models.graph_model import GraphKGE + if not isinstance(model, GraphKGE): + if (model.num_entities, model.num_relations) != (context.num_entities, context.num_relations): + raise ValueError('Transductive source must use the fixed backbone vocabulary') + yield + return + old = QueryContext.from_model(model) if model.graph_triples is not None else None + buffers = dict(model._buffers) + walk_graph = getattr(model, '_walk_graph', None) + sizes = model.num_entities, model.num_relations, model.num_direct_relations + try: + model.set_graph(context.triples, num_entities=context.num_entities, num_relations=context.num_relations, + inverse_relations=dict(context.inverse_relations)) + yield + finally: + if old is not None: + model.set_graph(old.triples, num_entities=old.num_entities, num_relations=old.num_relations, + inverse_relations=dict(old.inverse_relations)) + else: + for name, value in buffers.items(): + setattr(model, name, value) + if hasattr(model, '_walk_graph'): + model._walk_graph = walk_graph + model.num_entities, model.num_relations, model.num_direct_relations = sizes + model.clear_inference_cache() + + +@contextmanager +def evaluation_mode(model): + """Restore all module modes, including mixed train/eval configurations.""" + modes = [(module, module.training) for module in model.modules()] + if not any(training for _, training in modes): + yield + return + model.eval() + try: + yield + finally: + model.train(modes[0][1]) + for module, training in modes: + module.training = training diff --git a/dicee/query_answering/datasets.py b/dicee/query_answering/datasets.py new file mode 100644 index 000000000..95bef4ca1 --- /dev/null +++ b/dicee/query_answering/datasets.py @@ -0,0 +1,327 @@ +"""Load published UltraQuery and Is-CQA-Complex +H files without PyG. + +Graph splits and public inverse IDs follow DeepGraphLearning/ULTRA's +ultra/datasets_query.py at 427966ad8ed60420eef034063d44f3153addff90. +""" + +import hashlib +import pickle +import shutil +import tempfile +import urllib.request +import zipfile +from collections import Counter, defaultdict +from collections.abc import Sequence +from dataclasses import dataclass, field +from pathlib import Path + +import torch + +from ._query import QUERY_SHAPES, compile_query, index_query, nested +from .catalog import ( # noqa: F401 + BENCHMARK_DATASETS, + INDUCTIVE_VERSIONS, + IS_CQA, + PLUS_H_ARCHIVE, + PLUS_H_DATASETS, + PLUS_H_FOLDERS, + PLUS_H_PREFIX, + TRANSDUCTIVE, + ULTRA, + WIKITOPICS, + dataset_spec, + inference_graph_for_split, + query_types_for_dataset, +) +from .context import QueryContext, fingerprint + +REFERENCE_COMMIT = ULTRA[1] +PLUS_H_REFERENCE_COMMIT = IS_CQA[1] + + +def audit_plus_h_filters(root, data): + """Verify the authors' corrected filters; retain every root hard target. + + The complete graph is label-generation input only. It never leaves this + function or becomes the benchmark's inference context. + """ + if data.name not in PLUS_H_DATASETS or data.split != 'test': + raise ValueError('Filter correction is only defined for +H test labels') + folder = Path(root) / dataset_spec(data.name)[1] + graph = folder / 'KG_splits' if data.name == 'ICEWS18+H' else folder + files, triples = {}, [] + for split in ('train', 'valid', 'test'): + path = graph / f'{split}.txt' + payload = path.read_bytes() + files[path.relative_to(folder).as_posix()] = hashlib.sha256(payload).hexdigest() + triples.extend(tuple(map(int, line.split())) for line in payload.splitlines() if line.strip()) + full = QueryContext(triples, data.context.num_entities, data.context.num_relations, data.context.inverse_relations) + author_filters = {} + if data.name != 'ICEWS18+H': + for shape in ('2in', 'pni'): + selected = {q.query: q for q in data.queries if q.shape == shape} + if not selected: + continue + source = {} + for kind in ('queries', 'easy-answers', 'hard-answers'): + path = folder / 'test-query-reduction' / shape / 'all' / f'test-{kind}.pkl' + payload = path.read_bytes() + files[path.relative_to(folder).as_posix()] = hashlib.sha256(payload).hexdigest() + source[kind] = pickle.loads(payload) + queries = {q for group in source['queries'].values() for q in group} + if queries != selected.keys() or set(source['easy-answers']) != queries or set(source['hard-answers']) != queries: + raise ValueError('Author all/ files must contain exactly the selected query type') + for query, record in selected.items(): + if source['hard-answers'][query] != record.hard: + raise ValueError('Author all/ files changed the released hard targets') + author_filters[query] = frozenset(source['easy-answers'][query]) + changes, counts = {}, defaultdict(Counter) + for query in data.queries: + if 'n' not in query.shape: + continue + truth = full.answers(compile_query(query.query)) + if query.hard - truth: + raise ValueError(f'Released hard targets are not true in the complete graph: {data.name} {query.query}') + corrected = author_filters.get(query.query, query.easy) + if corrected != truth - query.hard: + raise ValueError(f'Author filters disagree with full-graph truth: {data.name} {query.query}') + remove, add = query.easy - corrected, corrected - query.easy + counts[query.shape].update(queries=1, hard_answers=len(query.hard), removed=len(remove), added=len(add), + changed_queries=int(bool(remove or add))) + if remove or add: + key = query.identity + changes[key] = dict(remove=sorted(remove), add=sorted(add)) + return dict(version=1, dataset=data.name, source_files=files, per_shape=dict(counts), changes=changes, + policy='Use author 2in/all and pni/all filters for FB/NELL; retain all root hard targets and other filters.') + + +def corrected_answer_filters(data, audit): + lookup = {q.identity: q for q in data.queries} + if set(audit['changes']) - lookup.keys(): + raise ValueError('Answer-filter audit does not match the benchmark queries') + return {lookup[key].query: (lookup[key].easy - set(change['remove'])) | set(change['add']) + for key, change in audit['changes'].items()} + + +@dataclass(frozen=True) +class BenchmarkQuery: + shape: str + query: tuple + easy: frozenset + hard: frozenset + + @property + def identity(self) -> str: + """Stable ID of the query and its released answers, used by plans, traces and audits.""" + return fingerprint((self.shape, self.query, sorted(self.easy), sorted(self.hard))) + + +@dataclass +class QueryBenchmark: + name: str + group: str + split: str + context: QueryContext + queries: tuple + candidates: tuple + metadata: dict = field(default_factory=dict) + entity_to_idx: dict | None = None + relation_to_idx: dict | None = None + + def __post_init__(self): + n, nr = self.context.num_entities, self.context.num_relations + candidates = set(self.candidates) + if not candidates or len(candidates) != len(self.candidates) or any(type(i) is not int or not 0 <= i < n for i in candidates): + raise ValueError('Benchmark candidates must be distinct valid entity IDs') + if not self.queries: + raise ValueError('Benchmark has no queries for the selected shapes') + seen = set() + for q in self.queries: + index_query(q.shape, q.query, range(n), range(nr)) + if q.query in seen: + raise ValueError('Duplicate benchmark query') + seen.add(q.query) + if not q.hard or q.easy & q.hard or not (q.easy | q.hard) <= candidates: + raise ValueError('Answers must be disjoint, include hard answers, and belong to the candidate domain') + + +def source_query_groups(data, root, shapes): + """Recover original per-type order for methods whose batches affect scores.""" + files = [name for name in data.metadata['files'] if name.endswith('queries.pkl')] + if len(files) != 1: + raise ValueError('Expected one source query file for upstream ordering') + with (Path(root) / dataset_spec(data.name)[1] / files[0]).open('rb') as stream: + structured = pickle.load(stream) + lookup = {query.query: query for query in data.queries} + return [[lookup[nested(query)] for query in structured[QUERY_SHAPES[shape]]] for shape in shapes] + + +def download_benchmark(name, root): + """Download an official archive once and extract its data under root.""" + _, folder, url = dataset_spec(name) + root = Path(root).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=root, prefix='.download-') as temporary: + archive = Path(temporary) / 'data.zip' + with urllib.request.urlopen(url, timeout=120) as response, archive.open('wb') as stream: + shutil.copyfileobj(response, stream) + with zipfile.ZipFile(archive) as zipped: + # Validate the complete archive before writing any member. + for member in zipped.infolist(): + target = (root / member.filename).resolve() + if not target.is_relative_to(root) or (member.external_attr >> 16) & 0o170000 == 0o120000: + raise ValueError('Unsafe path in dataset archive') + members = zipped.infolist() + if name in PLUS_H_DATASETS: + # This release also contains old benchmarks, training queries, + # and large stratified analyses. Extract only evaluation inputs + # for all three +H datasets, so the suite downloads only once. + prefixes = [f'{PLUS_H_PREFIX}/{folder}/' for folder in PLUS_H_FOLDERS.values()] + inputs = {'id2ent.pkl', 'id2rel.pkl', 'train.txt', 'valid.txt', 'test.txt', 'stats.txt', + *(f'{split}-{kind}.pkl' for split in ('valid', 'test') + for kind in ('queries', 'easy-answers', 'hard-answers')), + *(f'test-query-reduction/{shape}/all/test-{kind}.pkl' for shape in ('2in', 'pni') + for kind in ('queries', 'easy-answers', 'hard-answers'))} + members = [m for m in members for prefix in prefixes if m.filename.startswith(prefix) + and (m.filename[len(prefix):] in inputs or + m.filename[len(prefix):] in {'KG_splits/train.txt', 'KG_splits/valid.txt', 'KG_splits/test.txt'})] + zipped.extractall(root, members=members) + return root / folder + + +def load_benchmark(root: str | Path, name: str, *, split: str = 'test', query_types: Sequence[str] | None = None, + download: bool = False, inference_graph: str | None = None) -> QueryBenchmark: + """Load a released benchmark split from its official, trusted pickle files. + + Entity and relation IDs and answer labels are kept unchanged. Edges of the + evaluated split never enter the inference graph: +H test queries use + train+valid facts unless ``inference_graph='train'``, and validation queries + always use training facts. + + Args: + root: Directory into which the official archives are extracted. + name: A name from ``BENCHMARK_DATASETS`` or ``PLUS_H_DATASETS``. + split: ``'valid'`` or ``'test'``. + query_types: Query types to load (default: every type of the dataset). + download: Download and extract a missing archive into ``root``. + inference_graph: Facts for +H test queries, ``'train'`` or ``'train+valid'``. + + Returns: + The queries with their easy and hard answers, inference graph and candidates. + + Raises: + FileNotFoundError: If the dataset is missing and ``download`` is false. + ValueError: For an invalid split or query types, or inconsistent released files. + """ + if split not in ('valid', 'test'): + raise ValueError('Benchmark split must be valid or test') + group, folder, url = dataset_spec(name) + plus_h = name in PLUS_H_DATASETS + inference_graph = inference_graph_for_split(name, split, inference_graph) + expected_shapes = query_types_for_dataset(name) + shapes = tuple(expected_shapes if query_types is None else query_types) + if not shapes or len(set(shapes)) != len(shapes) or set(shapes) - set(expected_shapes): + raise ValueError('Select distinct supported query types for this benchmark (use 2u/up for DNF unions)') + path = Path(root).expanduser() / folder + if not path.is_dir(): + if not download: + raise FileNotFoundError(f'{path}: extract {url} under {root}, or pass download=True') + download_benchmark(name, root) + checksums = {} + + def source(filename): + file = path / filename + with file.open('rb') as stream: + checksums[filename] = hashlib.file_digest(stream, 'sha256').hexdigest() + return file + + def unpickle(filename): + with source(filename).open('rb') as stream: + return pickle.load(stream) + + def triples(stem): + if (path / f'{stem}.txt').is_file(): + with source(f'{stem}.txt').open() as stream: + return [tuple(map(int, line.split())) for line in stream if line.strip()] + return [tuple(row) for row in torch.load(source(f'{stem}.pt'), map_location='cpu', weights_only=True).tolist()] + + def nodes(edges): + return sorted({v for h, _, t in edges for v in (h, t)}) + + entities = relations = None + extended = name.startswith('InductiveFB15k237QueryExtendedEval:') + if group == 'transductive': + entities, relations = unpickle('id2ent.pkl'), unpickle('id2rel.pkl') + n, nr = len(entities), len(relations) + if set(entities) != set(range(n)) or set(relations) != set(range(nr)): + raise ValueError('Expected contiguous public vocabulary IDs') + graph_prefix = 'KG_splits/' if name == 'ICEWS18+H' else '' + edges = triples(graph_prefix + 'train') + if plus_h and split == 'test' and inference_graph == 'train+valid': + edges += triples(graph_prefix + 'valid') + candidates = list(range(n)) + pairs = tuple((r, r + 1) for r in range(0, nr, 2)) + structured = unpickle(f'{split}-queries.pkl') + easy, hard = unpickle(f'{split}-easy-answers.pkl'), unpickle(f'{split}-hard-answers.pkl') + else: + train = triples('train_graph') + if group == 'inductive-e': + valid, test = triples('val_inference'), triples('test_inference') + vocabulary = train + valid + test + n = max(nodes(vocabulary)) + 1 + nr = max(r for _, r, _ in vocabulary) + 1 + edges = train + (valid if split == 'valid' else test) + else: + edges = train if split == 'valid' else triples('test_inference') + n = max(nodes(edges)) + 1 + nr = max(r for _, r, _ in edges) + 1 + if nodes(edges) != list(range(n)): + raise ValueError('WikiTopics requires contiguous entity IDs') + candidates = nodes(edges) + pairs = tuple((r, r + nr // 2) for r in range(nr // 2)) + if group == 'inductive-er' and (path / 'og_mappings.pkl').is_file(): + # The full relation vocabulary can contain trailing unused IDs; + # max(observed ID) + 1 need not be twice the inverse offset. + mapping = unpickle('og_mappings.pkl')['r2id'] + pairs = tuple((r, mapping[label + '_inv']) for label, r in mapping.items() + if label + '_inv' in mapping and r < nr and mapping[label + '_inv'] < nr) + elif nr % 2: + raise ValueError('A relation mapping is required for an incomplete reciprocal vocabulary') + structured = unpickle('train_queries.pkl' if extended else f'{split}_queries.pkl') + easy = None if extended else unpickle(f'{split}_answers_easy.pkl') + hard = unpickle(f'train_answers_{split}.pkl' if extended else f'{split}_answers_hard.pkl') + if group == 'transductive' and nr % 2: + raise ValueError('Official benchmarks require paired direct/inverse relations') + # These archives already contain reciprocals. Fail rather than silently + # changing the inference graph if their documented convention does not hold. + context = QueryContext(edges, n, nr, pairs) + if set(context.triples) != set(edges): + raise ValueError('Dataset graph is missing reciprocal edges or uses unexpected inverse IDs') + records, available = [], set() + for structure, queries in structured.items(): + shape = next((key for key, value in QUERY_SHAPES.items() if value == nested(structure)), None) + if shape not in shapes: + continue + available.add(shape) + for i, query in enumerate(queries): + if group == 'transductive': + e, h = easy[query], hard[query] + elif extended: + e, h = (), hard[structure][i] + else: + e, h = easy[structure][query], hard[structure][query] + records.append(BenchmarkQuery(shape, nested(query), frozenset(e), frozenset(h))) + if set(shapes) - available: + raise ValueError(f'Missing requested query types: {sorted(set(shapes) - available)}') + records.sort(key=lambda q: (q.shape, q.query)) + metadata = dict(reference_commit=PLUS_H_REFERENCE_COMMIT if plus_h else REFERENCE_COMMIT, + suite='plus-h' if plus_h else 'ultraquery', files=checksums, url=url, + query_types=list(shapes), expected_query_types=list(expected_shapes), + evaluation='hardness-balanced-answers' if plus_h else 'faithfulness' if extended else 'hard-answers') + if plus_h: + metadata.update(paper='https://arxiv.org/abs/2410.12537', release='benchs-1.0', + inference_graph='train+valid' if split == 'test' and inference_graph == 'train+valid' else 'train', + answer_filter='published easy answers, including non-selected inference answers') + return QueryBenchmark(name, group, split, context, tuple(records), tuple(candidates), metadata, + {v: k for k, v in entities.items()} if entities else None, + {v: k for k, v in relations.items()} if relations else None) diff --git a/dicee/query_answering/engine.py b/dicee/query_answering/engine.py new file mode 100644 index 000000000..86cf56cc0 --- /dev/null +++ b/dicee/query_answering/engine.py @@ -0,0 +1,442 @@ +"""Shared beam and exact query execution over complete atomic rows.""" + +import math +import weakref +from collections import OrderedDict +from collections.abc import Hashable, Sequence +from contextlib import contextmanager + +import torch + +from ..models._inference import to_device +from ._deferred import deferred_checks, require +from ._query import atomic_conditions, compile_query, execute_query, positive, validate_tree +from .context import QueryContext, evaluation_mode, fingerprint, state_token +from .score_adapter import QueryScoreAdapter + +# Public contexts derived from graph models, released with the model. +_MODEL_CONTEXTS = weakref.WeakKeyDictionary() +_UNSET = object() + + +def _model_context(model, token, tensors): + """Share one derived context among all scorers of a model (e.g. adapter controls). + + The entry holds the graph tensors themselves, so a reused object ID cannot match. + """ + shared = _MODEL_CONTEXTS.get(model) + if shared is None or shared[0] != token or any(a is not b for a, b in zip(shared[1], tensors)): + shared = _MODEL_CONTEXTS[model] = token, tensors, QueryContext.from_model(model) + return shared[2] + + +class AtomicBatchCache: + """Share bounded logits across adapters without changing batch geometry. + + ``model`` keeps scores on their producing device, avoiding host round trips + when GPU memory is available. CPU storage remains the portable default. + ``granularity='row'`` stores single rows and computes only missing rows, in + batches of their own. Rows are then reused whatever batch requests them, so + a row's last bits can depend on the batch that first computed it. + """ + + def __init__(self, max_bytes, *, device='cpu', granularity='batch'): + if type(max_bytes) is not int or max_bytes < 0: + raise ValueError('Atomic cache size must be a nonnegative integer') + if device not in ('cpu', 'model'): + raise ValueError('Atomic cache device must be cpu or model') + if granularity not in ('batch', 'row'): + raise ValueError('Atomic cache granularity must be batch or row') + self.max_bytes = max_bytes + self.device = device + self.granularity = granularity + self.rows, self.token, self.used = OrderedDict(), None, 0 + self.hits, self.computed, self.peak = 0, 0, 0 + + def clear(self): + self.rows.clear() + self.used, self.token = 0, None + + def get(self, token, conditions): + if token is None or token != self.token: + self.clear() + self.token = token + key = tuple(conditions) + if token is not None and key in self.rows: + self.rows.move_to_end(key) + self.hits += len(conditions) + return self.rows[key] + return None + + def put(self, conditions, values): + self.computed += len(conditions) + size = values.numel() * values.element_size() + if self.token is None or size > self.max_bytes: + return + key = tuple(conditions) + if key in self.rows: + old = self.rows.pop(key) + self.used -= old.numel() * old.element_size() + while self.rows and self.used + size > self.max_bytes: + _, old = self.rows.popitem(last=False) + self.used -= old.numel() * old.element_size() + target = values.device if self.device == 'model' else 'cpu' + self.rows[key] = values.detach().to(target, copy=True) + self.used += size + self.peak = max(self.peak, self.used) + + def lookup(self, token: Hashable | None, conditions: Sequence[Hashable]) -> tuple[dict[Hashable, torch.Tensor], list[Hashable]]: + """Row granularity: cached rows by condition and the distinct missing conditions.""" + if token is None or token != self.token: + self.clear() + self.token = token + found, missing = {}, [] + for key in dict.fromkeys(conditions): + if token is not None and key in self.rows: + self.rows.move_to_end(key) + found[key] = self.rows[key] + else: + missing.append(key) + self.hits += sum(key in found for key in conditions) + return found, missing + + def put_rows(self, conditions: Sequence[Hashable], values: torch.Tensor) -> None: + """Row granularity: store each row of a computed batch on its own.""" + self.computed += len(conditions) + if self.token is None or not len(conditions): + return + size = values[0].numel() * values.element_size() + if size > self.max_bytes: + return + stored = values.detach().to(values.device if self.device == 'model' else 'cpu') + for key, row in zip(conditions, stored): + if key in self.rows: + self.used -= self.rows.pop(key).numel() * row.element_size() + while self.rows and self.used + size > self.max_bytes: + _, old = self.rows.popitem(last=False) + self.used -= old.numel() * old.element_size() + # A copy per row lets eviction release each row's memory. + self.rows[key] = row.clone() + self.used += size + self.peak = max(self.peak, self.used) + + +class AtomicScorer: + """Use DICE's existing all-tail API; only stochastic scoring needs special care.""" + + def __init__(self, model, context=None, *, row_batch_size=8, seed=0, samples=None, raw_cache=None): + from ..models.graph_model import GraphKGE, RelationGraphKGE + if isinstance(model, RelationGraphKGE) or getattr(model, 'relation_prediction', False) or getattr(model, 'name', None) == 'Shallom': + raise ValueError('Entity query answering requires an entity-prediction model') + if type(row_batch_size) is not int or row_batch_size < 1 or (samples is not None and (type(samples) is not int or samples < 1)) or type(seed) is not int: + raise ValueError('Positive row/sample counts and an integer seed are required') + self.model, self.explicit_context = model, context + self.graph_model = isinstance(model, GraphKGE) + self.row_batch_size, self.seed, self.samples = row_batch_size, seed, samples + self.raw_cache = raw_cache + self.refresh() + + def refresh(self): + self.n, self.nr = self.model.num_entities, self.model.num_relations + if not isinstance(self.n, int) or not isinstance(self.nr, int) or min(self.n, self.nr) < 1: + raise ValueError('Query answering requires a complete, positive entity/relation vocabulary') + attached = None + if self.graph_model: + from ..models._fused_message import tensor_version + self.model._require_graph() + tensors = (self.model.graph_triples, self.model.relation_id_map) + token = (self.n, self.nr, self.model.num_direct_relations, + *((id(t), tensor_version(t)) for t in tensors)) + cacheable = all(not t.is_inference() for t in tensors) + if not cacheable or token != getattr(self, '_context_token', None): + self._attached_context = (_model_context(self.model, token, tensors) if cacheable + else QueryContext.from_model(self.model)) + self._context_token = token + attached = self._attached_context + if attached is not None and self.explicit_context is not None and attached.identity != self.explicit_context.identity: + raise ValueError('Observed context must match the graph attached to the model') + self.context = attached if self.explicit_context is None else self.explicit_context + if self.context is not None and (self.context.num_entities, self.context.num_relations) != (self.n, self.nr): + raise ValueError('Context and model vocabularies disagree') + + @property + def device(self): + return next(self.model.parameters(), torch.empty(0)).device + + @contextmanager + def evaluation(self): + """Enter evaluation mode once for a query, including all scoring batches.""" + if getattr(self, '_evaluating', False): + yield + return + with evaluation_mode(self.model): + self._evaluating, self._token_memo = True, _UNSET + try: + yield + finally: + self._evaluating, self._token_memo = False, _UNSET + + def cache_token(self): + """Walking the model's tensors is costly, so one evaluation context walks them once. + + This assumes parameters, buffers and the attached graph do not change inside + the context; a caller that mutates them there must leave and re-enter it. + """ + if getattr(self, '_evaluating', False): + if self._token_memo is _UNSET: + self._token_memo = self._cache_token() + return self._token_memo + return self._cache_token() + + def _cache_token(self): + from ..models._inference import float32_precision_token + from ..models.flock import Flock + # Training callers can still reuse rows within a query, but never carry + # them across predictions. Autocast and unversioned tensors also opt out. + if any(m.training for m in self.model.modules()) or torch.is_autocast_enabled(self.device.type): + return None + token = self.model.inference_token() if self.graph_model else state_token(self.model) + if token is None: + return None + sampling = None + if isinstance(self.model, Flock): + sampling = (self.seed, self.model.test_samples if self.samples is None else self.samples, + self.model.walk_num, self.model.walk_len, self.model.refinements, + self.model.compact_state, self.model.pack_walks, self.model.compile_sampler) + return (id(self.model), token, self.n, self.nr, self.context.identity if self.context else None, + self.row_batch_size, getattr(self.model, 'query_batch_size', None), sampling, + float32_precision_token(), torch.are_deterministic_algorithms_enabled()) + + @torch.no_grad() + def rows(self, conditions): + conditions = [tuple(map(int, pair)) for pair in conditions] + if not conditions: + return torch.empty((0, self.n), device=self.device) + if any(not (0 <= h < self.n and 0 <= r < self.nr) for h, r in conditions): + raise ValueError('Atomic query outside the public vocabulary') + token = self.cache_token() if self.raw_cache is not None else None + with self.evaluation(): + if self.raw_cache is not None and self.raw_cache.granularity == 'row': + return self._cached_rows(token, conditions) + cached = self.raw_cache.get(token, conditions) if self.raw_cache is not None else None + if cached is not None: + return cached.to(self.device, copy=True) + result = self._compute(conditions) + if self.raw_cache is not None: + self.raw_cache.put(conditions, result) + return result.detach() + + def _cached_rows(self, token, conditions): + """Row-granular reuse: compute only rows missing from the cache, then restore the requested order.""" + found, missing = self.raw_cache.lookup(token, conditions) + fresh = {} + if missing: + computed = self._compute(missing) + self.raw_cache.put_rows(missing, computed) + fresh = {key: i for i, key in enumerate(missing)} + reused = [i for i, key in enumerate(conditions) if key not in fresh] + if not reused: + return computed[[fresh[key] for key in conditions]].detach() + # Stacking copies, so callers never alias cached storage. + rows = torch.stack([found[conditions[i]] for i in reused]).to(self.device) + if not fresh: + return rows + result = torch.empty((len(conditions), self.n), dtype=computed.dtype, device=self.device) + result[reused] = rows + new = [i for i, key in enumerate(conditions) if key in fresh] + result[new] = computed[[fresh[conditions[i]] for i in new]] + return result.detach() + + def _compute(self, conditions): + from ..models.flock import Flock + from ..models.graph_model import GraphKGE + if isinstance(self.model, Flock): + samples = self.model.test_samples if self.samples is None else self.samples + seeds = [int(fingerprint([self.context.identity, self.seed, samples, h, r])[:15], 16) for h, r in conditions] + result = self.model.forward_k_vs_all_seeded(torch.tensor(conditions, device=self.device), seeds, samples=samples) + else: + # Graph models validate and map host queries before an asynchronous copy. + host = self.graph_model and type(self.model).forward_k_vs_sample is GraphKGE.forward_k_vs_sample + batches = (torch.tensor(conditions[start:start + self.row_batch_size]) + for start in range(0, len(conditions), self.row_batch_size)) + result = torch.cat([self.model.forward_k_vs_all(batch if host else to_device(batch, self.device)) + for batch in batches]) + if result.shape != (len(conditions), self.n): + raise ValueError('Atomic scorer must return finite complete [rows, entities] logits') + require(torch.isfinite(result).all(), ValueError('Atomic scorer must return finite complete [rows, entities] logits')) + return result + + +class QueryAnswerer: + """Answer indexed query tuples with the same evaluator for KGEs and KGFMs. + + Graph models supply their attached context automatically. For a transductive + model, pass QueryContext to use context features or observed memberships. + ``predict`` returns memberships (or log-memberships with ``return_log_scores``). + Evaluation-mode predictions share a bounded row cache, invalidated by graph, + weight, adapter or inference-setting changes. Training callers cache only + within a prediction. ``last_info`` reports reuse and whether a beam pruned. + ``restore_observed=False`` disables final proof restoration while retaining + the adapter's atomic observed-fact setting. + """ + + def __init__(self, model, *, context=None, adapter=None, observed_mix=None, + row_batch_size=8, cache_bytes=64 * 1024 * 1024, seed=0, samples=None, raw_cache=None, + restore_observed=True): + if cache_bytes < 0: + raise ValueError('Cache size must be nonnegative') + if type(restore_observed) is not bool: + raise ValueError('Observed-answer restoration must be a boolean') + self.scorer = AtomicScorer(model, context, row_batch_size=row_batch_size, seed=seed, samples=samples, raw_cache=raw_cache) + if adapter is not None and observed_mix is not None and observed_mix != adapter.observed_mix: + raise ValueError('Set observed_mix on the adapter instead of overriding it') + self.adapter = adapter if adapter is not None else QueryScoreAdapter('global', observed_mix or 0.) + self.custom_adapter = adapter is not None + self.restore_observed = restore_observed + self.cache_bytes = cache_bytes + self.last_info = {} + self.clear_cache() + + def clear_cache(self): + """Release rows, also required after custom non-tensor scoring changes.""" + self._cache, self._cache_used, self._cache_token = OrderedDict(), 0, None + + def _prepare_cache(self, use_logits): + token, adapter_token = self.scorer.cache_token(), state_token(self.adapter) + if token is None and self.scorer.raw_cache is not None: + self.scorer.raw_cache.clear() + token = ((token, adapter_token, tuple(self.adapter.configuration.items()), use_logits) + if token is not None and adapter_token is not None else None) + if token is None or token != self._cache_token: + self.clear_cache() + self._cache_token = token + while self._cache and self._cache_used > self.cache_bytes: + _, old = self._cache.popitem(last=False) + self._cache_used -= old.numel() * old.element_size() + + def _condition_batches(self, conditions, use_logits, stats): + cache = self._cache + for start in range(0, len(conditions), self.scorer.row_batch_size): + batch = conditions[start:start + self.scorer.row_batch_size] + ready, missing = {}, [] + for key in dict.fromkeys(batch): + if key in cache: + ready[key] = cache[key] + cache.move_to_end(key) + stats['cache_hits'] += 1 + else: + missing.append(key) + if missing: + raw = self.scorer.rows(missing).to(torch.float64) + stats['raw_rows'] += len(missing) + if use_logits: + transformed = raw + else: + context = self.scorer.context + obs, base = context.features(missing, device=raw.device) if context and self.adapter.needs_context else (None, None) + transformed = self.adapter(raw, obs, base) + for key, value in zip(missing, transformed): + # Copies keep evicted batches from being retained by one row. + value = value.clone() + ready[key] = value + size = value.numel() * value.element_size() + if size <= self.cache_bytes: + while self._cache_used + size > self.cache_bytes: + _, old = cache.popitem(last=False) + self._cache_used -= old.numel() * old.element_size() + cache[key] = value + self._cache_used += size + # Stacking also isolates returned scores from mutable cache storage. + yield batch, torch.stack([ready[key] for key in batch]) + + def _row_batches(self, heads, relation, use_logits, stats): + for batch, values in self._condition_batches([(head, relation) for head in heads], use_logits, stats): + yield [head for head, _ in batch], values + + @torch.no_grad() + def prefetch(self, queries, *, use_logits=False): + """Warm reusable anchor rows without inspecting answer labels.""" + self.scorer.refresh() + self.adapter.verify_model(self.scorer.model) + self._prepare_cache(use_logits) + stats = dict(raw_rows=0, cache_hits=0) + if self._cache_token is None or self.cache_bytes < self.scorer.n * 8: + return stats + conditions = [] + for query in queries: + tree = compile_query(query) + validate_tree(tree, self.scorer.n, self.scorer.nr) + conditions.extend(atomic_conditions(tree)) + missing = [key for key in dict.fromkeys(conditions) if key not in self._cache] + missing = missing[:self.cache_bytes // (self.scorer.n * 8)] + with self.scorer.evaluation(), self._checked(): + for _ in self._condition_batches(missing, use_logits, stats): + pass + return stats + + @contextmanager + def _checked(self): + """Read device-side validity checks once; drop rows cached before a failure.""" + try: + with deferred_checks(): + yield + except BaseException: + self.clear_cache() + if self.scorer.raw_cache is not None: + self.scorer.raw_cache.clear() + raise + + @torch.no_grad() + def predict(self, query, *, beam_size=64, tnorm='prod', neg_norm='standard', lambda_=0., + use_logits=False, return_log_scores=False, executor='cqd', negation='model'): + """QTO evaluates every potentially improving head without a beam limit. + + ``negation='observed'`` negates from the known facts alone: a negated branch removes its exact answers over the + context graph and keeps every other entity at membership one (a control; ``'model'`` is the default). + """ + if executor not in ('cqd', 'qto'): + raise ValueError('Choose cqd or qto executor') + if negation not in ('model', 'observed'): + raise ValueError('Negation must be model or observed') + if negation == 'observed' and (use_logits or self.scorer.context is None): + raise ValueError('Negation from known facts needs memberships and a context graph') + if executor == 'qto' and use_logits: + raise ValueError('QTO requires bounded memberships, not raw logits') + if type(beam_size) is not int or beam_size < 1: + raise ValueError('beam_size must be a positive integer') + if tnorm not in ('prod', 'min') or neg_norm not in ('standard', 'sugeno', 'yager'): + raise ValueError('Unknown conjunction or negation norm') + if not math.isfinite(lambda_) or (neg_norm == 'sugeno' and lambda_ <= -1) or (neg_norm == 'yager' and lambda_ <= 0): + raise ValueError('Sugeno requires lambda_ > -1; Yager requires lambda_ > 0') + if use_logits and (self.custom_adapter or self.adapter.observed_mix or return_log_scores): + raise ValueError('Raw logits cannot be combined with adapters, observed overrides, or log-membership output') + self.scorer.refresh() + self.adapter.verify_model(self.scorer.model) + n, nr = self.scorer.n, self.scorer.nr + context = self.scorer.context + if context is None and self.adapter.needs_context: + raise ValueError('Adapter features and observed overrides require a context graph') + tree = compile_query(query) + validate_tree(tree, n, nr) + self._prepare_cache(use_logits) + stats = dict(raw_rows=0, cache_hits=0, pruned=False, negated_pruning=False, bound_skipped=0) + + # Skipped cache writes can change future miss batches; require scalar backend rows. + skip_zero_prefix = (not self.scorer.model.training and + getattr(self.scorer.model, 'query_batch_size', self.scorer.row_batch_size) == 1) + with self.scorer.evaluation(), self._checked(): + result, stats['pruned'] = execute_query( + tree, lambda heads, relation: self._row_batches(heads, relation, use_logits, stats), + n=n, device=self.scorer.device, row_batch_size=self.scorer.row_batch_size, + beam_size=beam_size, tnorm=tnorm, neg_norm=neg_norm, lambda_=lambda_, + negation_answers=context.answers if negation == 'observed' else None, + use_logits=use_logits, executor=executor, stats=stats, skip_zero_prefix=skip_zero_prefix) + if self.restore_observed and not use_logits and self.adapter.observed_mix == 1 and positive(tree): + result = result.clone() + answers = sorted(context.answers(tree)) + if answers: + result.index_fill_(0, to_device(torch.tensor(answers), result.device), 0.) + require(~(torch.isnan(result) | torch.isposinf(result)).any(), + FloatingPointError('Query composition produced invalid scores')) + self.last_info = dict(stats, search='global-prefix' if executor == 'cqd' else 'exact', cache_bytes=self._cache_used) + return result if use_logits or return_log_scores else result.exp() diff --git a/dicee/query_answering/method_evaluation.py b/dicee/query_answering/method_evaluation.py new file mode 100644 index 000000000..91758d65b --- /dev/null +++ b/dicee/query_answering/method_evaluation.py @@ -0,0 +1,391 @@ +"""Shared evaluation of checkpoint-compatible CQA methods and KGFM adapters.""" + +import time +from collections import Counter, defaultdict +from collections.abc import Callable, Mapping +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import Any, cast + +import torch + +from ._checkpoint import checksum, write_json +from ._query import QUERY_SHAPES, compile_query +from .benchmark import _accumulate_inference_statistics, _query_plan, evaluate_benchmark +from .catalog import GRAPH_INDEPENDENT_METHODS, METHODS, check_recipe, query_types_for_dataset # noqa: F401 +from .context import attached_context, evaluation_mode, fingerprint +from .datasets import QueryBenchmark, load_benchmark, source_query_groups +from .engine import AtomicBatchCache, QueryAnswerer +from .methods import CLMPT, REFERENCES, ConE, load_method +from .methods._common import FuzzyGNN +from .score_adapter import QueryScoreAdapter + + +def save_results(report: dict[str, Any], output: str | Path, *, run: dict[str, Any], metadata: dict[str, Any], + rank_trace_path: str | Path | None = None) -> dict[str, Any]: + """Finalize shared provenance, paired results, and rank checksums together.""" + output = Path(output) + trace = Path(rank_trace_path) if rank_trace_path is not None else output / 'ranks.sqlite3' + for name, comparison in report.pop('comparisons', {}).items(): + filter_control = comparison.pop('filter_control_of', None) + if filter_control is None: + comparison['paired_with'] = run['entry'] + else: + comparison['filter_paired_with'] = run['entry'] + ('-' + filter_control if filter_control else '') + if filter_control: + comparison['paired_with'] = run['entry'] + '-released-filters' + comparison.update(metadata, benchmark_run=dict(run, entry=f'{run["entry"]}-{name}'), + rank_trace_sha256=checksum(trace.parent / name / 'ranks.sqlite3')) + write_json(output / name / 'result.json', comparison) + report.setdefault('comparison_results', {})[name] = checksum(output / name / 'result.json') + report.update(metadata, benchmark_run=run, rank_trace_sha256=checksum(trace)) + write_json(output / 'result.json', report) + return report + + +def evaluate_method(entry: Mapping[str, Any], *, output: str | Path, device: str = 'cpu', split: str = 'test', + limit: int | None = None, seed: int = 0, threads: int | None = None, data: QueryBenchmark | None = None, + query_plan: list | None = None, batch_ends: list[int] | None = None, + rank_trace_path: str | Path | None = None, provenance: dict | None = None, + query_order: str = 'relation', write_result: bool = True, + on_prediction: Callable[[tuple, Any], None] | None = None, reference_batches: dict | None = None, + on_control_prediction: Callable[[tuple, Any], None] | None = None, + filter_corrections: Mapping[tuple, frozenset | set] | None = None) -> dict[str, Any]: + """Evaluate a published method or a KGFM adapter recipe with shared metrics and resumable progress. + + Args: + entry: Recipe with ``method``, ``checkpoint``, ``dataset``, ``root`` and + optional ``options``, ``query_types``, ``query_batch_size`` and + ``inference_graph``. Transductive embedding methods also bind + ``vocabulary_dataset``; KGFM adapters specify ``adapters``, + per-type ``operators`` and ``selection_protocol``. + output: Directory for ``result.json``, rank traces and progress. + device: Inference device. + split: ``'valid'`` or ``'test'``. + limit: Uniform sample of at most this many queries per type. + seed: Sampling seed. + threads: CPU threads for PyTorch. + data: Preloaded benchmark matching ``entry`` and ``split``. + query_plan: Explicit ordered queries. + batch_ends: Explicit batch boundaries in ``query_plan``. + rank_trace_path: Rank trace file (default: ``output/ranks.sqlite3``). + provenance: Identity of a frozen study, replacing the direct-run identity. + query_order: ``'relation'``, ``'canonical'`` or ``'upstream-pickle'``. + write_result: Write ``result.json`` with run metadata. + on_prediction: Called with each query and its scores. + reference_batches: Upstream batch of each query's first member. + on_control_prediction: Called for identity-adapter control predictions. + filter_corrections: Corrected easy answers by query. + + Returns: + The evaluation report. + + Raises: + ValueError: If the recipe or supplied data are invalid. + """ + if entry.get('method') not in METHODS: + raise ValueError(f'Choose a supported query method: {METHODS}') + if data is not None and (data.name != entry.get('dataset') or data.split != split): + raise ValueError('Loaded benchmark does not match the requested dataset and split') + if data is not None and entry.get('inference_graph') is not None: + expected_graph = entry['inference_graph'] if split == 'test' else 'train' + if data.metadata.get('inference_graph') != expected_graph: + raise ValueError('Loaded benchmark does not match the requested inference graph') + kgfm = entry['method'] not in REFERENCES + required = {'method', 'checkpoint', 'dataset', 'root', *(('adapters', 'operators', 'selection_protocol') if kgfm else ())} + allowed = required | {'options', 'query_types', 'query_batch_size', 'id', 'inference_graph', + *(('adapter_ablation',) if kgfm else ('vocabulary_dataset', 'per_shape'))} + if required - entry.keys() or entry.keys() - allowed: + raise ValueError(f'Expected fields {sorted(required)}; unexpected fields {sorted(entry.keys() - allowed)}') + types = entry.get('query_types') + if types is not None and (not types or set(types) - QUERY_SHAPES.keys() or len(types) != len(set(types))): + raise ValueError('Invalid query type selection') + check_recipe(dict(entry), types or query_types_for_dataset(entry['dataset'])) + output = Path(output) + rank_trace_path = Path(rank_trace_path) if rank_trace_path is not None else output / 'ranks.sqlite3' + if query_plan is None and query_order == 'canonical': + query_order = 'published' + if query_plan is None and query_order == 'upstream-pickle': + data = data if data is not None else load_benchmark(entry['root'], entry['dataset'], split=split, + query_types=entry.get('query_types'), inference_graph=entry.get('inference_graph')) + selected = set(_query_plan(data, limit, 'published', sampling='uniform', seed=seed)) + shapes = entry.get('query_types') or list(dict.fromkeys(query.shape for query in data.queries)) + size = entry.get('query_batch_size', 1) + query_plan, batch_ends, reference_batches = [], [], dict(reference_batches or {}) + for group in source_query_groups(data, entry['root'], shapes): + for start in range(0, len(group), size): + batch = group[start:start + size] + probes = [query for query in batch if query in selected] + if probes: + reference_batches[probes[0].query] = [query.query for query in batch] + query_plan.extend(probes) + batch_ends.append(len(query_plan)) + limit = None + direct_run = dict(entry=entry.get('id', f'{entry["method"]}-{entry["dataset"]}'), + phase='test' if split == 'test' else 'pilot', mode='direct') + direct_metadata = {'reference': {'status': 'direct evaluation; no frozen-study verification'}} + if entry['method'] in REFERENCES: + if on_control_prediction is not None: + raise ValueError('Identity calibration controls require a KGFM adapter method') + report = _evaluate_checkpoint_method( + entry, output=output, device=device, split=split, limit=limit, seed=seed, threads=threads, + data=data, query_plan=query_plan, batch_ends=batch_ends, rank_trace_path=rank_trace_path, + provenance=provenance, query_order=query_order, + on_prediction=on_prediction, reference_batches=reference_batches, filter_corrections=filter_corrections) + return save_results(report, output, run=direct_run, metadata=direct_metadata, + rank_trace_path=rank_trace_path) if write_result else report + options = entry.get('options', {}) + if threads is not None: + if threads < 1: + raise ValueError('Thread count must be positive') + torch.set_num_threads(threads) + from ..models._inference import float32_precision_backends + for backend in float32_precision_backends(): + backend.fp32_precision = 'ieee' + started = time.monotonic() + data = data if data is not None else load_benchmark(entry['root'], entry['dataset'], split=split, + query_types=entry.get('query_types'), inference_graph=entry.get('inference_graph')) + identity = provenance if provenance is not None else dict( + manifest=entry, checkpoint_sha256=checksum(entry['checkpoint']), + adapters={name: checksum(path) for name, path in entry['adapters'].items()}, + device=str(device), torch=str(torch.__version__), threads=torch.get_num_threads(), seed=seed) + report: dict[str, Any] = _evaluate_kgfm( + dict(entry, options=options), data, query_plan, batch_ends, output, device, identity, on_prediction, reference_batches, + on_control_prediction, limit=limit, seed=seed, query_order=query_order, rank_trace_path=rank_trace_path, + filter_corrections=filter_corrections) + report['total_wall_seconds'] = time.monotonic() - started + report['manifest_sha256'] = fingerprint(entry) + if write_result: + save_results(report, output, run=direct_run, metadata=direct_metadata, rank_trace_path=rank_trace_path) + return report + + +def _evaluate_checkpoint_method(entry, *, output, device='cpu', split='test', limit=None, seed=0, threads=None, + data=None, query_plan=None, batch_ends=None, rank_trace_path=None, provenance=None, query_order='relation', + on_prediction=None, reference_batches=None, filter_corrections=None): + """One dataset/model at a time; both tie protocols share every prediction.""" + method, dataset, types = entry['method'], entry['dataset'], entry.get('query_types') + if method not in ('ultraquery', 'ultraquery-lp', 'incoming-relation') and entry.get('vocabulary_dataset') != dataset: + raise ValueError('Bind transductive checkpoint IDs explicitly with vocabulary_dataset equal to dataset') + if threads is not None: + if threads < 1: + raise ValueError('Thread count must be positive') + torch.set_num_threads(threads) + torch.set_float32_matmul_precision('highest') + load_started = time.monotonic() + data = data if data is not None else load_benchmark(entry['root'], dataset, split=split, query_types=types, + inference_graph=entry.get('inference_graph')) + options = dict(entry.get('options', {})) + if method == 'incoming-relation': + options.setdefault('seed', seed) + model = load_method(method, entry['checkpoint'], data.context, device=device, **options) + shape_by_query = {item.query: item.shape for item in data.queries} + overrides = entry.get('per_shape', {}) + if set(overrides) - set(shape_by_query.values()): + raise ValueError('Per-shape configuration names must occur in the selected benchmark') + permitted = {'beam_size', 'tnorm', 'max_k'} if method in ('cqd', 'cqd-hybrid') else set() + if any(set(value) - permitted for value in overrides.values()): + raise ValueError('Only CQD beam_size, tnorm and max_k permit per-shape overrides') + defaults = {key: getattr(model, key) for key in permitted} + for value in overrides.values(): + if ('tnorm' in value and value['tnorm'] not in ('prod', 'min') or + any(key in value and (type(value[key]) is not int or value[key] < 1) for key in ('beam_size', 'max_k'))): + raise ValueError('Invalid per-shape CQD configuration') + batch_size = entry.get('query_batch_size', 1) + prepared = {} + + def prepare(queries): + prepared.clear() + if isinstance(model, (CLMPT, ConE, FuzzyGNN)): + requested = set(queries) + queries = (reference_batches or {}).get(queries[0], queries) + groups = defaultdict(list) + for query in queries: + groups[shape_by_query[query]].append(query) + for group in groups.values(): + if isinstance(model, FuzzyGNN): + scores = model.predict_batch(group) + prepared.update({query: scores[i] for i, query in enumerate(group) if query in requested}) + else: + branches = model.encode_trees([compile_query(query) for query in group]) + prepared.update({query: [branch[i:i + 1] for branch in branches] + for i, query in enumerate(group) if query in requested}) + + def predict(query): + if query in prepared: + value = prepared.pop(query) + scores = value if isinstance(model, FuzzyGNN) else cast(CLMPT | ConE, model).score_embeddings(value)[0] + else: + for key, value in (defaults | overrides.get(shape_by_query[query], {})).items(): + setattr(model, key, value) + scores = model.predict(query) + if on_prediction: + on_prediction(query, scores) + return scores + + identity = dict(**model.provenance, manifest=entry, torch=torch.__version__, device=device, + cuda=torch.version.cuda, threads=torch.get_num_threads(), matmul_precision='highest', + batch_size=batch_size) + if provenance is not None: + identity['paper_protocol'] = provenance + load_seconds = time.monotonic() - load_started + destination = Path(output) + destination.mkdir(parents=True, exist_ok=True) + report = evaluate_benchmark(data, predict, tie_policy='sort', additional_tie_policies=('expected',), + max_queries_per_shape=limit, query_sampling='uniform', sampling_seed=seed, + query_order=query_order, query_batch_size=batch_size, prepare=prepare, + query_plan=query_plan, batch_ends=batch_ends, rank_trace_path=rank_trace_path, + checkpoint_dir=destination / 'progress', checkpoint_identity=identity, + filter_corrections=filter_corrections, + checkpoint_every=100, on_checkpoint=lambda report: write_json(destination / 'partial.json', report)) + report['inference'] = identity + for comparison in report.get('comparisons', {}).values(): + comparison['inference'] = identity + report['load_seconds'] = load_seconds + report['manifest_sha256'] = fingerprint(entry) + return report + + +@contextmanager +def deterministic_kgfm(): + previous = torch.are_deterministic_algorithms_enabled() + warn_only = torch.is_deterministic_algorithms_warn_only_enabled() + torch.use_deterministic_algorithms(True) + try: + yield + finally: + torch.use_deterministic_algorithms(previous, warn_only=warn_only) + + +def _evaluate_kgfm(entry, data, plan, ends, output, device, identity, on_prediction=None, reference_batches=None, + on_control_prediction=None, *, limit=None, seed=0, query_order='relation', rank_trace_path=None, + filter_corrections=None): + from ..models import KGICL, TRIX, ULTRA + from ..models.flock import Flock + backbone = entry['method'].split('-')[0] + options = entry['options'] + allowed = {'beam_size', 'row_batch_size', 'backend_batch_size', 'cache_bytes', 'raw_cache_bytes', 'observed_facts', 'observed_ties', + 'raw_cache_device', 'raw_cache_granularity', 'relation_cache_mb', 'projection_cache_mb', 'negation'} + if backbone == 'kgicl': + allowed |= {'prompt_seed'} + if backbone == 'ultra': + allowed |= {'relation_conditioning'} + if backbone == 'flock': + allowed |= {'test_samples', 'walk_num'} + if options.keys() - allowed: + raise ValueError('Unknown KGFM execution option') + if options.get('raw_cache_device', 'cpu') not in ('cpu', 'model'): + raise ValueError('Atomic cache device must be cpu or model') + granularity = options.get('raw_cache_granularity', 'batch') + if granularity not in ('batch', 'row'): + raise ValueError('Atomic cache granularity must be batch or row') + # KG-ICL samples prompt graphs from the attached context with a fixed seed (default 0); + # options without a default here keep each backbone's construction unchanged. + extra = {'kgicl_prompt_seed': options['prompt_seed']} if 'prompt_seed' in options else {} + if 'relation_conditioning' in options: + extra['ultra_relation_conditioning'] = options['relation_conditioning'] + if backbone == 'flock': + # Flock samples walks; each atom row has its own seed (AtomicScorer), so scores do not depend on batching. + extra.update(flock_test_samples=options.get('test_samples', 1), flock_walk_num=options.get('walk_num', 128)) + model = {'ultra': ULTRA, 'trix': TRIX, 'kgicl': KGICL, 'flock': Flock}[backbone](dict( + num_entities=1, num_relations=1, graph_inference_backend='auto', + graph_relation_cache_mb=options.get('relation_cache_mb', 64), + graph_projection_cache_mb=options.get('projection_cache_mb', 64), + **{f'{backbone}_query_batch_size': options.get('backend_batch_size', 1)}, **extra)).load_pretrained( + entry['checkpoint']).to(device).eval().requires_grad_(False) + adapters = {name: QueryScoreAdapter.load(path, model=model) for name, path in entry['adapters'].items()} + for operator, adapter in adapters.items(): + expected = 'prod' if operator.partition(':')[0] == 'product' else 'min' + if adapter.metadata.get('training', {}).get('tnorm', 'prod') != expected: + raise ValueError('Operator must use an adapter fitted with the matching t-norm') + observed_facts = options.get('observed_facts') + if observed_facts is not None: + for adapter in adapters.values(): + adapter.observed_mix = float(observed_facts != 'none') + restore_observed = observed_facts in (None, 'both') + # Ties among known facts in a beam: lowest entity IDs (the default) or the backbone's order. + ties = options.get('observed_ties', 'entity') + if ties not in ('entity', 'model'): + raise ValueError('Observed ties must be entity or model') + for adapter in adapters.values(): + adapter.observed_tie_break = ties == 'model' + # Negated branches: 1 - membership (the default) or the known facts alone (a control). + negation = options.get('negation', 'model') + if negation not in ('model', 'observed'): + raise ValueError('Negation must be model or observed') + shape_by_query = {query.query: query.shape for query in data.queries} + stats = Counter() + paired = entry.get('adapter_ablation', False) + # Row granularity also helps a single configuration: beams of different queries share rows. + raw_cache = AtomicBatchCache(options.get('raw_cache_bytes', 2 * 1024**3), device=options.get('raw_cache_device', 'cpu'), + granularity=granularity) if paired or granularity == 'row' else None + with ExitStack() as stack: + stack.enter_context(deterministic_kgfm()) + stack.enter_context(attached_context(model, data.context)) + stack.enter_context(evaluation_mode(model)) + recipes = {'learned': adapters} + if paired: + recipes['without-adapter'] = {name: QueryScoreAdapter('global', adapter.observed_mix, + normalization=adapter.normalization, metadata={'calibration': 'identity'}) for name, adapter in adapters.items()} + for control in recipes['without-adapter'].values(): + control.observed_tie_break = ties == 'model' + engines = {variant: {name: QueryAnswerer(model, context=data.context, adapter=adapter, + restore_observed=restore_observed, + row_batch_size=options.get('row_batch_size', 1), + raw_cache=raw_cache, + cache_bytes=options.get('cache_bytes', 512 * 1024**2) // (len(adapters) * len(recipes))) + for name, adapter in recipe.items()} for variant, recipe in recipes.items()} + all_engines = [engine for group in engines.values() for engine in group.values()] + + def record(info): + _accumulate_inference_statistics(stats, info, all_engines) + if raw_cache is not None: + stats['backbone_computed_rows'] = initial_raw['computed'] + raw_cache.computed + stats['backbone_reused_rows'] = initial_raw['reused'] + raw_cache.hits + stats['raw_cache_peak_bytes'] = max(stats.get('raw_cache_peak_bytes', 0), raw_cache.peak) + + initial_raw = {} + + def initialize_raw_counts(): + if not initial_raw: + initial_raw.update(computed=stats.get('backbone_computed_rows', 0), reused=stats.get('backbone_reused_rows', 0)) + + def prepare(queries): + initialize_raw_counts() + queries = (reference_batches or {}).get(queries[0], queries) + for group in engines.values(): + for operator, engine in group.items(): + selected = [query for query in queries if entry['operators'][shape_by_query[query]] == operator] + if selected: + record(engine.prefetch(selected)) + + def predict(query, variant='learned'): + operator = entry['operators'][shape_by_query[query]] + engine = engines[variant][operator] + result = engine.predict(query, beam_size=options.get('beam_size', 64), + tnorm='prod' if operator.partition(':')[0] == 'product' else 'min', return_log_scores=True, + negation=negation) + record(engine.last_info) + callback = on_prediction if variant == 'learned' else on_control_prediction + if callback: + callback(query, result) + return result + + report = evaluate_benchmark(data, predict, additional_tie_policies=('expected',), query_plan=plan, + prepare=prepare, statistics=stats, query_order=query_order, + query_batch_size=entry.get('query_batch_size', 1), max_queries_per_shape=limit, + query_sampling='uniform', sampling_seed=seed, + batch_ends=ends, checkpoint_dir=output / 'progress', checkpoint_identity=identity, + checkpoint_every=100, rank_trace_path=rank_trace_path, + filter_corrections=filter_corrections, + on_checkpoint=lambda report: write_json(output / 'partial.json', report), + comparison_predictors={'without-adapter': lambda query: predict(query, 'without-adapter')} if paired else None) + for variant, result in [('learned', report), *report.get('comparisons', {}).items()]: + calibration = result.get('filter_control_of', variant) or 'learned' + result['inference'] = dict(method=entry['method'], options=options, operators=entry['operators'], deterministic_algorithms=True, + calibration=calibration, selection_protocol=entry['selection_protocol'], cache_statistics=dict(stats), + observed_facts=dict(mode=observed_facts or 'checkpoint-default', + atomic_mix={name: adapter.observed_mix for name, adapter in adapters.items()}, + restore_answers=restore_observed), + runtime_scope='paired adapter evaluation' if paired else 'single configuration') + return report diff --git a/dicee/query_answering/methods/NOTICE.md b/dicee/query_answering/methods/NOTICE.md new file mode 100644 index 000000000..00aea2795 --- /dev/null +++ b/dicee/query_answering/methods/NOTICE.md @@ -0,0 +1,77 @@ +# Method sources + +Upstream revisions are pinned in [checkpoints.py](checkpoints.py). + + +## ConE + +MIT License + +Copyright (c) 2021 ConE (Cone Embeddings for Multi-Hop Reasoning over Knowledge Graphs) Authors (https://arxiv.org/abs/2110.13715) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +## GNN-QE + +MIT License + +Copyright (c) 2022 MilaGraph + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +## ULTRA / UltraQuery + +MIT License + +Copyright (c) 2023 MilaGraph + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/dicee/query_answering/methods/__init__.py b/dicee/query_answering/methods/__init__.py new file mode 100644 index 000000000..16ae0c124 --- /dev/null +++ b/dicee/query_answering/methods/__init__.py @@ -0,0 +1,12 @@ +"""Native inference methods with independent, published model semantics.""" + +from .checkpoints import REFERENCES, load_method +from .clmpt import CLMPT +from .cone import ConE +from .cqd import CQD +from .gnnqe import GNNQE +from .heuristic import IncomingRelationHeuristic +from .qto import QTO +from .ultraquery import UltraQuery + +__all__ = ['CLMPT', 'CQD', 'ConE', 'GNNQE', 'IncomingRelationHeuristic', 'QTO', 'UltraQuery', 'REFERENCES', 'load_method'] diff --git a/dicee/query_answering/methods/_bounded_attention.py b/dicee/query_answering/methods/_bounded_attention.py new file mode 100644 index 000000000..a67019755 --- /dev/null +++ b/dicee/query_answering/methods/_bounded_attention.py @@ -0,0 +1,36 @@ +"""Bound attention workspace without splitting coupled query sequences.""" + +from types import FunctionType + +import torch +from torch.nn import functional as F +from torch.overrides import TorchFunctionMode + + +def _attention_forward(mode): + function = F.multi_head_attention_forward + scope = function.__globals__ | { + 'scaled_dot_product_attention': lambda *args, **kwargs: mode.__torch_function__( + F.scaled_dot_product_attention, (), args, kwargs)} + return FunctionType(function.__code__, scope, function.__name__, function.__defaults__, function.__closure__) + + +class BoundedAttention(TorchFunctionMode): + def __init__(self, max_pairs=1024**2): + self.max_pairs = max_pairs + + def __torch_function__(self, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + if func is F.multi_head_attention_forward: + # The outer functional call suspends this mode inside its Python body. + return _attention_forward(self)(*args, **kwargs) + if func is F.scaled_dot_product_attention: + query, key, value = args[:3] + if query.is_cuda and query.shape[-3] > 1 and query.shape[-2] * key.shape[-2] > self.max_pairs: + mask = args[3] if len(args) > 3 else kwargs.get('attn_mask') + dropout = args[4] if len(args) > 4 else kwargs.get('dropout_p', 0) + if mask is not None or dropout != 0 or kwargs.get('enable_gqa', False): + raise ValueError('Bounded inference attention requires no mask or dropout') + return torch.cat([func(query[..., i:i+1, :, :], key[..., i:i+1, :, :], value[..., i:i+1, :, :], *args[3:], **kwargs) + for i in range(query.shape[-3])], dim=-3) + return func(*args, **kwargs) diff --git a/dicee/query_answering/methods/_common.py b/dicee/query_answering/methods/_common.py new file mode 100644 index 000000000..e1ae001a6 --- /dev/null +++ b/dicee/query_answering/methods/_common.py @@ -0,0 +1,229 @@ +"""Shared inference contracts for native, checkpoint-compatible methods.""" + +from collections import OrderedDict +from functools import reduce +from typing import Any + +import torch +from torch import nn + +from ...models._inference import float32_precision_token +from .._query import compile_query, validate_tree +from ..context import evaluation_mode, state_token +from ..score_adapter import SOFTMAX_DEGREE_CEILING + + +class QueryMethod(nn.Module): + provenance: dict[str, Any] # Checkpoint identity, set by ``load_method``. + + def __init__(self, context): + super().__init__() + self.context = context + + @property + def device(self): + return next(self.parameters()).device + + @torch.no_grad() + def predict(self, query): + tree = compile_query(query) + validate_tree(tree, self.context.num_entities, self.context.num_relations) + with evaluation_mode(self): + scores = self.score_tree(tree) + if scores.shape != (self.context.num_entities,) or not torch.isfinite(scores).all(): + raise ValueError('QueryMethod returned invalid scores') + return scores.clone() + + +def cache_token(model): + """Validity token of cached inference tensors; None when nothing may be cached (inference tensors, autocast).""" + token = state_token(model) + if torch.is_autocast_enabled(next(model.parameters()).device.type): + return None + return None if token is None else (token, float32_precision_token()) + + +class TensorCache: + """Bounded inference cache, invalidated by parameter and device changes.""" + + def __init__(self, capacity): + if type(capacity) is not int or capacity < 0: + raise ValueError('Cache capacity must be a nonnegative byte count') + self.capacity, self.bytes, self.token = capacity, 0, None + self.values = OrderedDict() + self.optional = OrderedDict() + + def refresh(self, model): + token = cache_token(model) + if token is None or token != self.token: + self.values.clear() + self.optional.clear() + self.bytes, self.token = 0, token + + def get(self, key): + value = self.values.get(key) + if value is not None: + self.values.move_to_end(key) + if key in self.optional: + self.optional.move_to_end(key) + return value + + def put(self, key, value, *, optional=False): + """Optional entries use spare bytes and are evicted before primary entries.""" + size = value.numel() * value.element_size() + if self.token is None or size > self.capacity: + return + previous = self.values.get(key) + previous_size = 0 if previous is None else previous.numel() * previous.element_size() + if optional and self.bytes - previous_size + size > self.capacity: + return + previous = self.values.pop(key, None) + self.optional.pop(key, None) + if previous is not None: + self.bytes -= previous.numel() * previous.element_size() + while self.bytes + size > self.capacity: + if self.optional: + victim, _ = self.optional.popitem(last=False) + old = self.values.pop(victim) + else: + _, old = self.values.popitem(last=False) + self.bytes -= old.numel() * old.element_size() + self.values[key] = value.detach().clone() + self.bytes += size + if optional: + self.optional[key] = None + + +def fuzzy_combine(values, op, logic): + if logic == 'product': + fn = (lambda a, b: a * b) if op == 'and' else (lambda a, b: a + b - a * b) + elif logic == 'godel': + fn = torch.minimum if op == 'and' else torch.maximum + elif logic == 'lukasiewicz': + fn = (lambda a, b: (a + b - 1).clamp_min(0)) if op == 'and' else (lambda a, b: (a + b).clamp_max(1)) + else: + raise ValueError('Choose product, godel or lukasiewicz logic') + return reduce(fn, values) + + +class FuzzyGNN(QueryMethod): + """Fuzzy-set execution with a learned projection. + + ``observed_traversal`` (off by default, and not part of any upstream method) also traverses the observed facts + exactly at every projection: a tail receives at least the highest membership of a head linked to it by the + projected relation, as with symbolic traversal (max over heads of a product with one). + + ``calibration='softmax-degree'`` (off by default, not part of any upstream method) replaces the sigmoid of every + projection by QTO's calibration, generalized to fuzzy head sets: the softmax of the projection logits over all + entities times d, the membership mass of the observed tails that the traversal reaches (at least one), capped at + 0.9999. For a single anchor head, d is the number of its observed tails, as in the beam executor's calibration. + """ + + def __init__(self, context, logic='product', observed_traversal=False, calibration=None): + super().__init__(context) + fuzzy_combine([torch.tensor(0.)], 'and', logic) + if type(observed_traversal) is not bool: + raise ValueError('observed_traversal must be a boolean') + if calibration not in (None, 'softmax-degree'): + raise ValueError('calibration must be None or softmax-degree') + self.logic = logic + self.observed_traversal = observed_traversal + self.calibration = calibration + self._relation_edges = None + + def membership(self, tree): + return self.membership_batch([tree])[0] + + def score_tree(self, tree): + probability = self.membership(tree) + return ((probability + 1e-10) / (1 - probability + 1e-10)).log() + + def membership_batch(self, trees): + op = trees[0][0] + if any(tree[0] != op or len(tree) != len(trees[0]) for tree in trees): + raise ValueError('GNN batches must share a query structure') + if op == 'anchor': + heads = torch.tensor([tree[1] for tree in trees], device=self.device) + return torch.nn.functional.one_hot(heads, self.context.num_entities).float() + if op == 'project': + relation_ids = tuple(tree[1] for tree in trees) + relations = torch.tensor(relation_ids, device=self.device) + source = self.membership_batch([tree[2] for tree in trees]) + if self.calibration is None: + projected = self.project_batch(source, relations, relation_ids=relation_ids) + return torch.maximum(projected, self.traverse(source, relations)) if self.observed_traversal else projected + known = self.traverse(source, relations) + logits = self.project_batch(source, relations, relation_ids=relation_ids, logits=True) + degree = known.sum(1, keepdim=True).clamp_min(1) + projected = (logits.log_softmax(1) + degree.log()).exp().clamp_max(SOFTMAX_DEGREE_CEILING) + return torch.maximum(projected, known) if self.observed_traversal else projected + if op == 'not': + return 1 - self.membership_batch([tree[1] for tree in trees]) + return fuzzy_combine([self.membership_batch(children) for children in zip(*(tree[1:] for tree in trees))], op, self.logic) + + def traverse(self, membership, relations): + """Exact traversal of observed facts: each tail's highest membership among heads with (head, relation, tail).""" + device = membership.device + if self._relation_edges is None or self._relation_edges[0].device != device: + triples = torch.tensor(self.context.triples, dtype=torch.long).reshape(-1, 3) + triples = triples[torch.argsort(triples[:, 1], stable=True)] + counts = torch.bincount(triples[:, 1], minlength=self.context.num_relations) + offsets = torch.cat((counts.new_zeros(1), counts.cumsum(0))) + self._relation_edges = tuple(t.contiguous().to(device) for t in (offsets, triples[:, 0], triples[:, 2])) + offsets, heads, tails = self._relation_edges + starts = offsets[relations] + lengths = offsets[relations + 1] - starts + result = torch.zeros_like(membership) + total = int(lengths.sum()) + if total: + rows = torch.repeat_interleave(torch.arange(len(relations), device=device), lengths, output_size=total) + positions = torch.arange(total, device=device) - (lengths.cumsum(0) - lengths)[rows] + starts[rows] + values = membership[rows, heads[positions]] + # A maximum does not depend on the reduction order, so this is deterministic on every device. + result.view(-1).scatter_reduce_(0, rows * membership.shape[1] + tails[positions], values, 'amax') + return result + + @torch.no_grad() + def predict_batch(self, queries): + trees = [compile_query(query) for query in queries] + if not trees: + return next(self.parameters()).new_empty((0, self.context.num_entities)) + for tree in trees: + validate_tree(tree, self.context.num_entities, self.context.num_relations) + with evaluation_mode(self): + probability = self.membership_batch(trees) + scores = ((probability + 1e-10) / (1 - probability + 1e-10)).log() + if not torch.isfinite(scores).all(): + raise ValueError('GNN returned invalid scores') + return scores + + +def union_branches(tree): + """Distribute projection over DNF unions for embedding query models.""" + if tree[0] == 'or': + return [branch for child in tree[1:] for branch in union_branches(child)] + if tree[0] == 'project': + return [('project', tree[1], child) for child in union_branches(tree[2])] + if any(child[0] == 'or' for child in tree[1:] if isinstance(child, tuple)): + raise ValueError('Only benchmark DNF unions are supported') + return [tree] + + +def complex_rows(model, heads, relations): + """ComplEx with the released CQD/QTO two-matmul evaluation order.""" + heads = torch.as_tensor(heads, device=model.entity_embeddings.weight.device, dtype=torch.long) + relations = torch.as_tensor(relations, device=heads.device, dtype=torch.long) + hr, hi = model.entity_embeddings(heads).chunk(2, -1) + rr, ri = model.relation_embeddings(relations).chunk(2, -1) + tr, ti = model.entity_embeddings.weight.chunk(2, -1) + return (hr * rr - hi * ri) @ tr.T + (hi * rr + hr * ri) @ ti.T + + +def require_complex(model, context): + from ...models.complex import ComplEx + if not isinstance(model, ComplEx): + raise TypeError('The released method requires a ComplEx backbone') + if (model.num_entities, model.num_relations) != (context.num_entities, context.num_relations): + raise ValueError('Checkpoint and graph vocabularies must match') + if model.embedding_dim % 2: + raise ValueError('ComplEx needs equal real and imaginary dimensions') diff --git a/dicee/query_answering/methods/_ordered_aggregation.py b/dicee/query_answering/methods/_ordered_aggregation.py new file mode 100644 index 000000000..34b1f8af1 --- /dev/null +++ b/dicee/query_answering/methods/_ordered_aggregation.py @@ -0,0 +1,52 @@ +"""Reference-order sparse reductions for checkpoint reproduction.""" + +import torch + + +def cached_layout(cache, name, edges, types, nodes, *, relation_order=False): + # Inference tensors have no version counter, so their layout cannot be cached safely. + if edges.is_inference() or types.is_inference(): + return layout(edges, types, nodes, relation_order=relation_order) + token = edges.device, edges.data_ptr(), edges._version, types.data_ptr(), types._version + previous = cache.get(name) + if previous is None or previous[0] != token: + cache[name] = token, layout(edges, types, nodes, relation_order=relation_order) + return cache[name][1] + + +def layout(edges, types, nodes, *, relation_order=False): + key = edges[0] * (edges[1].max() + 1) + edges[1] if types.numel() else types + if relation_order and types.numel(): + key = key * (types.max() + 1) + types + order = key.argsort() + rows, sources = edges[:, order] + pointers = torch.cat((rows.new_zeros(1), rows.bincount(minlength=nodes).cumsum(0))) + return pointers, sources, types[order] + + +def aggregate(states, relations, edges, reduction='sum', *, num_warps=1): + """Reduce neighbours sequentially, matching the upstream CUDA RSPMM loop. + + ``pna`` returns (sum, sum of squares, max, min) from one pass over the edges; + squares equal separate reductions of ``states.square()`` and ``relations.square()``. + Warps only change scheduling: each feature is still accumulated by one lane. + """ + if states.is_cuda: + from ._ordered_aggregation_cuda import ordered_reduce + return ordered_reduce(states, relations, edges, reduction, num_warps=num_warps) + if reduction == 'pna': + return (aggregate(states, relations, edges), aggregate(states.square(), relations.square(), edges), + aggregate(states, relations, edges, 'max'), aggregate(states, relations, edges, 'min')) + from ._ordered_aggregation_cpu import ordered_reduce + result = ordered_reduce(states, relations, edges, reduction) + if result is not None: + return result + pointers, sources, types = edges + fill = 0 if reduction == 'sum' else -torch.finfo(states.dtype).max if reduction == 'max' else torch.finfo(states.dtype).max + output = torch.full_like(states, fill) + for row in range(len(states)): + for edge in range(int(pointers[row]), int(pointers[row + 1])): + value = states[sources[edge]] * relations[types[edge]] + output[row] = (output[row] + value if reduction == 'sum' else + torch.maximum(output[row], value) if reduction == 'max' else torch.minimum(output[row], value)) + return output diff --git a/dicee/query_answering/methods/_ordered_aggregation_cpu.py b/dicee/query_answering/methods/_ordered_aggregation_cpu.py new file mode 100644 index 000000000..a210c2ae5 --- /dev/null +++ b/dicee/query_answering/methods/_ordered_aggregation_cpu.py @@ -0,0 +1,116 @@ +"""Small optional native CPU inference loop (no PyTorch extension build). + +Compile lazily with the system C compiler, without fast math or contraction. The +loop is single threaded and releases the GIL. Missing compilers use the literal +PyTorch implementation; training and unsupported shapes/dtypes do so as well. +The private temporary library lives for this process only. +""" + +import ctypes +import os +import shutil +import subprocess +import tempfile +import threading +from functools import lru_cache + +import torch + +_SOURCE = r''' +#include +#include +#include +#define DEFINE(NAME, TYPE, LIMIT) \ +void NAME(const TYPE *s, const TYPE *r, TYPE *o, \ + const int64_t *p, const int64_t *src, const int64_t *rel, \ + int64_t n, int64_t b, int64_t d, \ + int64_t sn, int64_t sb, int64_t sd, \ + int64_t rn, int64_t rb, int64_t rd, int kind) { \ + for (int64_t row = 0; row < n; ++row) { \ + for (int64_t batch = 0; batch < b; ++batch) { \ + TYPE *out = o + (row * b + batch) * d; \ + for (int64_t f = 0; f < d; ++f) \ + out[f] = kind == 0 ? 0 : kind == 1 ? -LIMIT : LIMIT; \ + for (int64_t e = p[row]; e < p[row + 1]; ++e) { \ + const TYPE *state = s + src[e] * sn + batch * sb; \ + const TYPE *relation = r + rel[e] * rn + batch * rb; \ + for (int64_t f = 0; f < d; ++f) { \ + TYPE value = state[f * sd] * relation[f * rd]; \ + TYPE old = out[f]; \ + out[f] = kind == 0 ? old + value : \ + kind == 1 ? (isnan(old) || old >= value ? old : value) : \ + (isnan(old) || old <= value ? old : value); \ + } \ + } \ + } \ + } \ +} +DEFINE(reduce_float, float, FLT_MAX) +DEFINE(reduce_double, double, DBL_MAX) +''' +_BUILD_LOCK = threading.Lock() + + +@lru_cache(maxsize=1) +def _library(): + compiler = shutil.which('cc') + if compiler is None or os.name != 'posix': + return None + directory = tempfile.TemporaryDirectory(prefix='dicee-ordered-') + try: + source = os.path.join(directory.name, 'ordered.c') + binary = os.path.join(directory.name, 'ordered.so') + with open(source, 'w') as stream: + stream.write(_SOURCE) + subprocess.run([compiler, '-std=c11', '-O3', '-shared', '-fPIC', + '-fno-fast-math', '-ffp-contract=off', source, '-o', binary], + check=True, capture_output=True, timeout=30) + library = ctypes.CDLL(binary) + library._directory = directory + for name in ('reduce_float', 'reduce_double'): + function = getattr(library, name) + function.argtypes = [ctypes.c_void_p] * 6 + [ctypes.c_int64] * 9 + [ctypes.c_int] + function.restype = None + return library + except (OSError, subprocess.SubprocessError): + directory.cleanup() + return None + + +def ordered_reduce(states, relations, edges, reduction): + """Return a native result, or None when the literal implementation is needed.""" + if (states.device.type != 'cpu' or relations.device.type != 'cpu' + or states.dtype not in (torch.float32, torch.float64) + or relations.dtype != states.dtype or states.ndim != 3 or relations.ndim != 3 + or states.shape[1:] != relations.shape[1:] + or states.layout != torch.strided or relations.layout != torch.strided + or states.is_neg() or relations.is_neg() + or (torch.is_grad_enabled() and (states.requires_grad or relations.requires_grad)) + or reduction not in ('sum', 'max', 'min')): + return None + pointers, sources, types = edges + if any(t.device.type != 'cpu' or t.dtype != torch.int64 or t.ndim != 1 + or not t.is_contiguous() for t in edges): + return None + nodes, batch, dim = states.shape + # Native memory accesses require validated CSR bounds. Keep malformed inputs + # on the Python path so they retain PyTorch's indexing/error semantics. + if (len(pointers) != nodes + 1 or len(sources) != len(types) + or int(pointers[0]) != 0 or int(pointers[-1]) != len(sources) + or bool((pointers[1:] < pointers[:-1]).any()) + or (sources.numel() and (int(sources.min()) < 0 or int(sources.max()) >= nodes + or int(types.min()) < 0 or int(types.max()) >= len(relations)))): + return None + if not states.numel(): + return torch.empty(states.shape, dtype=states.dtype, device=states.device) + with _BUILD_LOCK: + library = _library() + if library is None: + return None + output = torch.empty(states.shape, dtype=states.dtype, device=states.device) + function = library.reduce_float if states.dtype == torch.float32 else library.reduce_double + function(states.data_ptr(), relations.data_ptr(), output.data_ptr(), + pointers.data_ptr(), sources.data_ptr(), types.data_ptr(), + nodes, batch, dim, *states.stride(), *relations.stride(), + ('sum', 'max', 'min').index(reduction)) + return output diff --git a/dicee/query_answering/methods/_ordered_aggregation_cuda.py b/dicee/query_answering/methods/_ordered_aggregation_cuda.py new file mode 100644 index 000000000..cebc47fae --- /dev/null +++ b/dicee/query_answering/methods/_ordered_aggregation_cuda.py @@ -0,0 +1,106 @@ +"""CSR inference kernel preserving upstream neighbour accumulation order.""" + +import torch +import triton +import triton.language as tl +from triton.language.extra.cuda import libdevice + +# Rows above this multiple of the mean degree launch first, so their long serial +# chains overlap the remaining rows instead of trailing the launch. +HUB_FACTOR = 8 +# Independent edge loads issued ahead of the strictly sequential accumulation. +UNROLL = 8 + + +@triton.jit +def _reduce(S, R, O, O2, O3, O4, PTR, SRC, REL, ORDER, HUBS, ROWS, + SN: tl.constexpr, SB: tl.constexpr, SD: tl.constexpr, + RN: tl.constexpr, RB: tl.constexpr, RD: tl.constexpr, + B: tl.constexpr, D: tl.constexpr, WIDTH: tl.constexpr, KIND: tl.constexpr, UNROLL: tl.constexpr): + # Hubs for every batch element first; the remaining rows stay batch-major + # so concurrently running programs share one batch element's states in L2. + program = tl.program_id(0) + if program < HUBS * B: + rank = program // B + batch = program % B + else: + rest = program - HUBS * B + batch = rest // tl.maximum(ROWS - HUBS, 1) + rank = HUBS + rest % tl.maximum(ROWS - HUBS, 1) + row = tl.load(ORDER + rank) + feature = tl.arange(0, WIDTH) + present = feature < D + start, end = tl.load(PTR + row), tl.load(PTR + row + 1) + value = tl.full((WIDTH,), 0. if KIND == 'sum' or KIND == 'pna' else + -3.4028234663852886e38 if KIND == 'max' else 3.4028234663852886e38, tl.float32) + squares = tl.full((WIDTH,), 0., tl.float32) + high = tl.full((WIDTH,), -3.4028234663852886e38, tl.float32) + low = tl.full((WIDTH,), 3.4028234663852886e38, tl.float32) + for block in range(start, end, UNROLL): + # Loads do not depend on the running value, so they overlap; updates + # remain one edge at a time in CSR order, as in the upstream loop. + for offset in tl.static_range(UNROLL): + edge = block + offset + valid = edge < end + source = tl.load(SRC + edge, valid, other=0) + relation = tl.load(REL + edge, valid, other=0) + state = tl.load(S + source * SN + batch * SB + feature * SD, present & valid, other=0) + rel = tl.load(R + relation * RN + batch * RB + feature * RD, present & valid, other=0) + # Explicit round-to-nearest operations also prevent packed-instruction + # contraction when a lane owns multiple features on newer GPUs. + message = libdevice.mul_rn(state, rel) + if KIND == 'sum': + value = tl.where(valid, libdevice.add_rn(value, message), value) + elif KIND == 'max': + value = tl.where(valid, tl.maximum(value, message), value) + elif KIND == 'min': + value = tl.where(valid, tl.minimum(value, message), value) + else: + # PNA reads each edge once; squares match separate h*h and r*r inputs. + square = libdevice.mul_rn(libdevice.mul_rn(state, state), libdevice.mul_rn(rel, rel)) + value = tl.where(valid, libdevice.add_rn(value, message), value) + squares = tl.where(valid, libdevice.add_rn(squares, square), squares) + high = tl.where(valid, tl.maximum(high, message), high) + low = tl.where(valid, tl.minimum(low, message), low) + offset = (row * B + batch) * D + feature + tl.store(O + offset, value, present) + if KIND == 'pna': + tl.store(O2 + offset, squares, present) + tl.store(O3 + offset, high, present) + tl.store(O4 + offset, low, present) + + +def _schedule(pointers): + """Rows by launch rank, hubs first, cached on the CSR pointer tensor.""" + version = None if pointers.is_inference() else pointers._version + cached = getattr(pointers, '_dicee_schedule', None) + if cached is not None and version is not None and cached[0] == version: + return cached[1:] + degree = pointers[1:] - pointers[:-1] + hubs = degree > HUB_FACTOR * degree.double().mean() + rows = torch.arange(len(degree), device=pointers.device) + order = torch.cat((rows[hubs][degree[hubs].argsort(descending=True, stable=True)], rows[~hubs])).contiguous() + result = order, int(hubs.sum()) + if version is not None: + pointers._dicee_schedule = (version, *result) + return result + + +def ordered_reduce(states, relations, edges, reduction, *, num_warps=1): + """Return one reduction, or (sum, sum of squares, max, min) for ``pna``.""" + if states.dtype != torch.float32 or relations.dtype != torch.float32: + raise ValueError('Reference sparse inference requires float32') + if num_warps not in (1, 2, 4, 8): + raise ValueError('Ordered aggregation warps must be 1, 2, 4 or 8') + nodes, batch, dim = states.shape + outputs = [torch.empty((nodes, batch, dim), device=states.device, dtype=states.dtype) + for _ in range(4 if reduction == 'pna' else 1)] + if nodes and batch: + order, hubs = _schedule(edges[0]) + targets = outputs + [outputs[0]] * (4 - len(outputs)) + with torch.cuda.device(states.device): + _reduce[(nodes * batch,)](states, relations, *targets, *edges, order, hubs, nodes, + *states.stride(), *relations.stride(), + batch, dim, triton.next_power_of_2(dim), reduction, UNROLL, + num_warps=num_warps, enable_fp_fusion=False) + return tuple(outputs) if reduction == 'pna' else outputs[0] diff --git a/dicee/query_answering/methods/checkpoints.py b/dicee/query_answering/methods/checkpoints.py new file mode 100644 index 000000000..7f6124349 --- /dev/null +++ b/dicee/query_answering/methods/checkpoints.py @@ -0,0 +1,216 @@ +"""Strict conversion of published inference checkpoints into native modules.""" + +import hashlib +import json +from pathlib import Path +from typing import Any + +import torch + +from ...models.complex import ComplEx +from ..catalog import REFERENCES +from ..context import QueryContext, state_fingerprint +from ._common import QueryMethod +from .clmpt import CLMPT +from .cone import ConE +from .cqd import CQD +from .gnnqe import GNNQE +from .heuristic import IncomingRelationHeuristic +from .qto import QTO +from .ultraquery import UltraQuery + + +class _GraphPayload: + """Data-only receiver for the graph buffers in released TorchDrug weights.""" + + +def read_state(path, *, graph_context=None, graph_metadata=None, inductive=False): + allowed = [(_GraphPayload, 'torchdrug.data.graph.Graph')] if graph_context is not None else [] + with torch.serialization.safe_globals(allowed): + value = torch.load(path, map_location='cpu', weights_only=True) + while isinstance(value, dict) and not all(isinstance(tensor, (torch.Tensor, _GraphPayload)) for tensor in value.values()): + keys = [key for key in ('model', 'model_state_dict', 'state_dict') if key in value] + if len(keys) != 1: + raise ValueError('Expected an unambiguous tensor state dictionary') + value = value[keys[0]] + if isinstance(value, dict): + value = dict(value) + if inductive and any(not isinstance(value.get(key), _GraphPayload) + for key in ('train_graph', 'valid_graph', 'test_graph')): + raise ValueError('Missing inductive checkpoint graph buffers') + for key, graph in list(value.items()): + if not isinstance(graph, _GraphPayload): + continue + if key not in (('train_graph', 'valid_graph', 'test_graph') if inductive else ('fact_graph', 'graph')): + raise ValueError(f'Unexpected TorchDrug graph buffer {key}') + _validate_graph_buffer(key, graph, graph_context, graph_metadata, inductive=inductive) + del value[key] + if inductive: + for key in ('valid_nodes', 'test_nodes'): + nodes = value.pop(key, None) + if (not isinstance(nodes, torch.Tensor) or nodes.dtype != torch.long or nodes.ndim != 1 + or (nodes < 0).any() or (nodes >= graph_context.num_entities).any() + or len(nodes.unique()) != len(nodes)): + raise ValueError(f'Invalid inductive checkpoint node buffer {key}') + if graph_metadata is not None: + graph_metadata[key] = dict(sha256=state_fingerprint({'nodes': nodes}), used_for_inference=False) + if not isinstance(value, dict) or not value or not all(isinstance(k, str) and isinstance(v, torch.Tensor) for k, v in value.items()): + raise ValueError('Expected a nonempty tensor state dictionary') + if all(key.startswith('module.') for key in value): + value = {key.removeprefix('module.'): tensor for key, tensor in value.items()} + if any(not torch.isfinite(tensor).all() for tensor in value.values()): + raise ValueError('Checkpoint contains nonfinite values') + return value + + +def _validate_graph_buffer(name, graph, context, metadata, *, inductive=False): + edges, weights = graph._edge_list, graph._edge_weight + if (not isinstance(edges, torch.Tensor) or not isinstance(weights, torch.Tensor) + or not 0 < int(graph.num_node) <= context.num_entities + or (not inductive and int(graph.num_node) != context.num_entities) + or int(graph.num_relation) != context.num_relations + or edges.ndim != 2 or edges.shape[1] != 3 or edges.dtype != torch.long + or weights.shape != (len(edges),) or not (weights == 1).all()): + raise ValueError('TorchDrug graph buffer does not match the public vocabulary / unit-edge protocol') + if (edges < 0).any() or (edges[:, :2] >= int(graph.num_node)).any() or (edges[:, 2] >= context.num_relations).any(): + raise ValueError('TorchDrug graph buffer contains out-of-vocabulary IDs') + if name in ('fact_graph', 'train_graph'): + observed = torch.tensor(context.triples, dtype=torch.long).reshape(-1, 3) + keys = ((observed[:, 0] * context.num_relations + observed[:, 1]) * context.num_entities + observed[:, 2]).sort().values + requested = (edges[:, 0] * context.num_relations + edges[:, 2]) * context.num_entities + edges[:, 1] + positions = torch.searchsorted(keys, requested) + if not len(keys) or (positions >= len(keys)).any() or not torch.equal(keys[positions], requested): + raise ValueError('Checkpoint training facts are absent from the supplied graph; verify entity/relation ID maps') + if metadata is not None: + metadata[name] = dict(edges=len(edges), sha256=state_fingerprint({'edges': edges}), + used_for_inference=False, training_facts_verified=name in ('fact_graph', 'train_graph')) + + +def _strip(state, prefixes, sentinel): + for prefix in prefixes: + if prefix + sentinel in state and all(key.startswith(prefix) for key in state): + return {key[len(prefix):]: tensor for key, tensor in state.items()} + raise ValueError(f'Unrecognized checkpoint layout: expected {sentinel}') + + +def _complex(state, context): + layouts = [('entity_embeddings.weight', 'relation_embeddings.weight'), + ('embeddings.0.weight', 'embeddings.1.weight'), + ('_entity_embedding.weight', '_relation_embedding.weight')] + for entity_key, relation_key in layouts: + if set(state) == {entity_key, relation_key}: + entities, relations = state[entity_key], state[relation_key] + if (entities.ndim != 2 or relations.ndim != 2 or entities.shape[1] != relations.shape[1] + or entities.shape[1] % 2 or entities.shape[0] != context.num_entities + or relations.shape[0] != context.num_relations): + raise ValueError('ComplEx checkpoint dimensions do not match the public vocabulary') + model = ComplEx(dict(model='ComplEx', num_entities=context.num_entities, num_relations=context.num_relations, + embedding_dim=entities.shape[1], normalization=None)).to(dtype=entities.dtype) + model.load_state_dict({'entity_embeddings.weight': entities, 'relation_embeddings.weight': relations}, strict=True) + return model + raise ValueError('Unsupported ComplEx checkpoint keys') + + +def load_method(method: str, path: str | Path, context: QueryContext, *, device: str | torch.device = 'cpu', + **config: Any) -> QueryMethod: + """Load a published method's released checkpoint into its native implementation. + + Every learned tensor loads strictly with ``weights_only=True``; entity and + relation IDs are never inferred or permuted. + + Args: + method: A name from ``REFERENCES``. + path: Released checkpoint, or the heuristic's specification file. + context: Inference graph whose IDs match the checkpoint. + device: Inference device. + **config: Method options, such as CQD's ``beam_size`` or QTO's ``threshold``. + + Returns: + The method in evaluation mode, with checkpoint provenance in ``provenance``. + + Raises: + ValueError: For unknown methods, or checkpoints that do not match. + """ + if method not in REFERENCES: + raise ValueError(f'Unknown method {method}') + path = Path(path) + graph_metadata = {} + if method == 'incoming-relation': + expected = dict(version=1, method=method, reference_commit=REFERENCES[method][1], + ranking='query-seeded random shuffle within Boolean classes') + if json.loads(path.read_text()) != expected: + raise ValueError('Invalid untrained heuristic specification') + model = IncomingRelationHeuristic(context, **config) + state = None + else: + state = read_state(path, graph_context=context if method in ('gnnqe', 'inductive-gnnqe') else None, + graph_metadata=graph_metadata, inductive=method == 'inductive-gnnqe') + if method == 'incoming-relation': + pass + elif method in ('cqd', 'cqd-hybrid', 'qto'): + backbone = _complex(state, context) + model = (QTO(backbone, context, **config) if method == 'qto' else + CQD(backbone, context, hybrid=method == 'cqd-hybrid', **config)) + elif method in ('ultraquery', 'ultraquery-lp'): + state = _strip(state, ('', 'model.model.', 'model.'), 'relation_model.layers.0.linear.weight') + dim = state['relation_model.layers.0.linear.weight'].shape[0] + layers = len({key.split('.')[2] for key in state if key.startswith('relation_model.layers.')}) + model = UltraQuery(context, dim=dim, num_layers=layers, **config) + model.load_state_dict(state, strict=True) + elif method == 'cone': + dim = state['entity_embedding'].shape[1] + projection_dim = state['cone_proj.layer1.weight'].shape[0] + projection_layers = len([key for key in state if key.startswith('cone_proj.layer') and key.endswith('.weight')]) - 1 + model = ConE(context, dim=dim, gamma=state['gamma'].item(), projection_dim=projection_dim, + projection_layers=projection_layers, **config) + model.load_state_dict(state, strict=True) + elif method == 'clmpt': + nbp = {key.removeprefix('nbp.'): tensor for key, tensor in state.items() if key.startswith('nbp.')} + backbone = _complex(nbp, context) + pre_norm = any(key.startswith('transformer.encoder.') for key in state) + prefix = 'transformer.encoder.layers.' if pre_norm else 'transformer.layers.' + hidden_key = prefix + ('0.ffn.layer1.weight' if pre_norm else '0.linear1.weight') + layers = len({key[len(prefix):].split('.')[0] for key in state if key.startswith(prefix)}) + model = CLMPT(backbone, context, pre_norm=pre_norm, layers=layers, hidden_dim=state[hidden_key].shape[0], **config) + translated = {key: tensor for key, tensor in state.items() if not key.startswith('nbp.')} + translated.update({'model.' + key: tensor for key, tensor in backbone.state_dict().items()}) + model.load_state_dict(translated, strict=True) + else: + state = _strip(state, ('', 'model.'), 'model.query.weight') + translated = {} + for key, tensor in state.items(): + if key.startswith('model.model.layers.'): + target = key.removeprefix('model.model.') + elif key.startswith(('model.query.', 'model.mlp.')): + target = key.removeprefix('model.') + else: + raise ValueError(f'Unexpected GNN-QE parameter {key}') + translated[target] = tensor + dim = translated['query.weight'].shape[1] + layers = len({key.split('.')[1] for key in translated if key.startswith('layers.')}) + mlp_layers = len({key.split('.')[2] for key in translated if key.startswith('mlp.layers.')}) + config.setdefault('concat_hidden', translated['mlp.layers.0.weight'].shape[1] != 2 * dim) + config.setdefault('layer_norm', 'layers.0.layer_norm.weight' in translated) + config.setdefault('dependent', 'layers.0.relation_linear.weight' in translated) + if translated['layers.0.linear.weight'].shape[1] == 13 * dim: + config.setdefault('aggregate', 'pna') + model = GNNQE(context, dim=dim, num_layers=layers, mlp_layers=mlp_layers, **config) + model.load_state_dict(translated, strict=True) + model.to(device).eval().requires_grad_(False) + repo, commit = REFERENCES[method] + with path.open('rb') as stream: + checksum = hashlib.file_digest(stream, 'sha256').hexdigest() + model.provenance = dict(method=method, reference=f'https://github.com/{repo}/tree/{commit}', + reference_commit=commit, checkpoint=str(path.resolve()), + checkpoint_sha256=checksum, + model_state_sha256=state_fingerprint(model), configuration=config, + public_ids='unchanged; checkpoint must use the dataset ID maps', + context_sha256=context.identity, checkpoint_graph_buffers=graph_metadata) + if isinstance(model, CLMPT): + model.provenance.update(post_norm_batch_coupling=not model.pre_norm, + dnf_up='separate grounded branches matching the released evaluation helper') + if isinstance(model, CQD) and model.atomic_negation: + model.provenance.update( + negation_reference='https://github.com/EdinburghNLP/adaptive-cqd/tree/642ce042708be6247087c09c780b9deb47e941d3', + negation_scope='CQD-A signed-atom negation with +H scoring') + return model diff --git a/dicee/query_answering/methods/clmpt.py b/dicee/query_answering/methods/clmpt.py new file mode 100644 index 000000000..e95c1bb1b --- /dev/null +++ b/dicee/query_answering/methods/clmpt.py @@ -0,0 +1,183 @@ +"""CLMPT inference with released transformer and ComplEx checkpoint layouts.""" + +from collections import defaultdict + +import torch +from torch import nn + +from .._query import compile_query, validate_tree +from ..context import evaluation_mode +from ._bounded_attention import BoundedAttention +from ._common import QueryMethod, require_complex, union_branches + + +class Attention(nn.Module): + def __init__(self, dim, heads=8): + super().__init__() + self.heads = heads + self.size = dim // heads + for name in ('linear_q', 'linear_k', 'linear_v', 'output_layer'): + setattr(self, name, nn.Linear(dim, dim, bias=False)) + + def forward(self, value): + batch, length, dim = value.shape + q, k, v = [getattr(self, name)(value).view(batch, length, self.heads, self.size).transpose(1, 2) + for name in ('linear_q', 'linear_k', 'linear_v')] + weights = ((q * self.size ** -.5) @ k.transpose(2, 3)).softmax(-1) + result = (weights @ v).transpose(1, 2).contiguous().view(batch, length, dim) + return self.output_layer(result) + + +class FeedForward(nn.Module): + def __init__(self, dim, hidden): + super().__init__() + self.layer1, self.layer2 = nn.Linear(dim, hidden), nn.Linear(hidden, dim) + + def forward(self, value): + return self.layer2(self.layer1(value).relu()) + + +class EncoderLayer(nn.Module): + def __init__(self, dim, hidden): + super().__init__() + self.self_attention_norm, self.ffn_norm = nn.LayerNorm(dim, eps=1e-6), nn.LayerNorm(dim, eps=1e-6) + self.self_attention, self.ffn = Attention(dim), FeedForward(dim, hidden) + + def forward(self, value): + value = value + self.self_attention(self.self_attention_norm(value)) + return value + self.ffn(self.ffn_norm(value)) + + +class Transformer(nn.Module): + def __init__(self, dim, hidden, layers): + super().__init__() + self.scale = dim ** .5 + self.encoder = nn.Module() + self.encoder.layers = nn.ModuleList([EncoderLayer(dim, hidden) for _ in range(layers)]) + self.encoder.last_norm = nn.LayerNorm(dim, eps=1e-6) + + def forward(self, value): + value = value * self.scale + for layer in self.encoder.layers: + value = layer(value) + return self.encoder.last_norm(value) + + +def query_graph(tree): + """Compile a DNF branch into ordered signed relation atoms.""" + atoms, anchors, variables = [], {}, ['f'] + + def visit(node, target): + op = node[0] + if op == 'and': + for child in node[1:]: + visit(child, target) + return + sign = 1 + if op == 'not': + node, sign = node[1], -1 + if node[0] != 'project': + raise ValueError('CLMPT requires signed relation atoms') + child = node[2] + if child[0] == 'anchor': + source = f's{len(anchors) + 1}' + anchors[source] = child[1] + else: + source = f'e{len(variables)}' + variables.append(source) + visit(child, source) + atoms.append((source, node[1], target, sign)) + + visit(tree, 'f') + return atoms, anchors, variables + + +class CLMPT(QueryMethod): + def __init__(self, model, context, *, hidden_dim=8192, layers=2, pre_norm=True, aggregation='mean', + depth_shift=0, candidate_batch_size=4096): + super().__init__(context) + require_complex(model, context) + dim = model.embedding_dim + if dim % 8 or min(hidden_dim, layers, candidate_batch_size) < 1: + raise ValueError('CLMPT needs dimensions divisible by eight and positive batch/layer sizes') + if aggregation not in ('mean', 'sum', 'max'): + raise ValueError('Unknown CLMPT aggregation') + self.model, self.pre_norm, self.aggregation = model, pre_norm, aggregation + self.depth_shift, self.candidate_batch_size = depth_shift, candidate_batch_size + if pre_norm: + self.transformer = Transformer(dim, hidden_dim, layers) + else: + layer = nn.TransformerEncoderLayer(dim, 8, hidden_dim, .1, 'relu') + self.transformer = nn.TransformerEncoder(layer, layers, nn.LayerNorm(dim), enable_nested_tensor=False) + for name in ('existential_embedding', 'universal_embedding', 'free_embedding'): + self.register_parameter(name, nn.Parameter(torch.rand(1, dim))) + + @staticmethod + def message(entity, relation, inverse=False): + real, imag = entity.chunk(2, -1) + rr, ri = relation.chunk(2, -1) + if inverse: + ri = -ri + return torch.cat((real * rr - imag * ri, real * ri + imag * rr), -1) + + def embed_batch(self, trees): + graphs = [query_graph(tree) for tree in trees] + atoms, anchors, variables = graphs[0] + def signature(graph): + return [(a, c, d) for a, _, c, d in graph[0]], list(graph[1]), graph[2] + if any(signature(graph) != signature(graphs[0]) for graph in graphs[1:]): + raise ValueError('CLMPT batches must share a query structure') + terms = {name: self.model.entity_embeddings(torch.tensor([graph[1][name] for graph in graphs], device=self.device)) + for name in anchors} + terms.update({name: self.free_embedding if name == 'f' else self.existential_embedding for name in variables}) + relations = [self.model.relation_embeddings(torch.tensor([graph[0][index][1] for graph in graphs], device=self.device)) + for index in range(len(atoms))] + for _ in range(max(1, len(variables) + self.depth_shift)): + messages = defaultdict(list) + for index, (source, _, target, sign) in enumerate(atoms): + embedding = relations[index] + messages[target].append(sign * self.message(terms[source], embedding)) + if source in variables: + messages[source].append(sign * self.message(terms[target], embedding, inverse=True)) + updated = dict(terms) + for name in variables: + # Keep the release's tensor layout, including post-norm batch coupling. + with BoundedAttention(): + value = self.transformer(torch.stack([*messages[name], terms[name].expand(len(trees), -1)], 1)) + updated[name] = (value.max(1).values if self.aggregation == 'max' else + value.sum(1) if self.aggregation == 'sum' else value.mean(1)) + terms = updated + return terms['f'] + + def embed(self, tree): + return self.embed_batch([tree]) + + def encode_trees(self, trees): + expanded = [union_branches(tree) for tree in trees] + if len({len(branches) for branches in expanded}) != 1: + raise ValueError('CLMPT batches must share a query structure') + return [self.embed_batch(trees) for trees in zip(*expanded)] + + def score_embeddings(self, branches): + return torch.cat([torch.stack([torch.cosine_similarity(branch[:, None], entities, dim=-1) + for branch in branches]).max(0).values + for entities in self.model.entity_embeddings.weight.split(self.candidate_batch_size)], dim=-1) + + def score_trees(self, trees): + return self.score_embeddings(self.encode_trees(trees)) + + @torch.no_grad() + def predict_batch(self, queries): + trees = [compile_query(query) for query in queries] + if not trees: + return self.model.entity_embeddings.weight.new_empty((0, self.context.num_entities)) + for tree in trees: + validate_tree(tree, self.context.num_entities, self.context.num_relations) + with evaluation_mode(self): + result = self.score_trees(trees) + if not torch.isfinite(result).all(): + raise ValueError('CLMPT returned invalid scores') + return result + + def score_tree(self, tree): + return self.score_trees([tree])[0] diff --git a/dicee/query_answering/methods/cone.py b/dicee/query_answering/methods/cone.py new file mode 100644 index 000000000..c21fd1d26 --- /dev/null +++ b/dicee/query_answering/methods/cone.py @@ -0,0 +1,130 @@ +"""ConE inference. Architecture: MIRALab-USTC/QE-ConE (see NOTICE.md).""" + +import math + +import torch +from torch import nn + +from ._common import QueryMethod, TensorCache, union_branches + + +class ConeProjection(nn.Module): + def __init__(self, dim, hidden_dim=1600, num_layers=2): + super().__init__() + self.num_layers = num_layers + self.layer0 = nn.Linear(hidden_dim, 2 * dim) + self.layer1 = nn.Linear(2 * dim, hidden_dim) + for index in range(2, num_layers + 1): + setattr(self, f'layer{index}', nn.Linear(hidden_dim, hidden_dim)) + + def forward(self, axis, aperture, relation_axis, relation_aperture): + value = torch.cat((axis + relation_axis, aperture + relation_aperture), -1) + for index in range(1, self.num_layers + 1): + value = getattr(self, f'layer{index}')(value).relu() + axis, aperture = self.layer0(value).chunk(2, -1) + return axis.tanh() * math.pi, (2 * aperture).tanh() * math.pi / 2 + math.pi / 2 + + +class ConeIntersection(nn.Module): + def __init__(self, dim): + super().__init__() + self.layer_axis1, self.layer_axis2 = nn.Linear(2 * dim, dim), nn.Linear(dim, dim) + self.layer_arg1, self.layer_arg2 = nn.Linear(2 * dim, dim), nn.Linear(dim, dim) + + def forward(self, axes, apertures): + endpoints = torch.cat((axes - apertures, axes + apertures), -1) + attention = self.layer_axis2(self.layer_axis1(endpoints).relu()).softmax(0) + x, y = (attention * axes.cos()).sum(0), (attention * axes.sin()).sum(0) + x = x.masked_fill(x.abs() < 1e-3, 1e-3) + axis = (y / x).atan() + axis = torch.where((x < 0) & (y >= 0), axis + math.pi, axis) + axis = torch.where((x < 0) & (y < 0), axis - math.pi, axis) + gate = self.layer_arg2(self.layer_arg1(endpoints).relu().mean(0)).sigmoid() + return axis, apertures.min(0).values * gate + + +class ConE(QueryMethod): + def __init__(self, context, *, dim, gamma, center_reg, projection_dim=1600, projection_layers=2, candidate_batch_size=4096, cache_bytes=16 * 2**20): + super().__init__(context) + if dim < 1 or candidate_batch_size < 1 or center_reg < 0: + raise ValueError('Invalid ConE dimensions or center regularization') + self.angle_cache = TensorCache(cache_bytes) + self._range_token = None + self.cen, self.candidate_batch_size = center_reg, candidate_batch_size + self.gamma = nn.Parameter(torch.tensor([float(gamma)]), requires_grad=False) + self.embedding_range = nn.Parameter(torch.tensor([(gamma + 2) / dim]), requires_grad=False) + self.modulus = nn.Parameter(self.embedding_range.detach().clone() / 2) + self.entity_embedding = nn.Parameter(torch.empty(context.num_entities, dim)) + self.axis_embedding = nn.Parameter(torch.empty(context.num_relations, dim)) + self.arg_embedding = nn.Parameter(torch.empty(context.num_relations, dim)) + self.cone_proj = ConeProjection(dim, projection_dim, projection_layers) + self.cone_intersection = ConeIntersection(dim) + for parameter in (self.entity_embedding, self.axis_embedding, self.arg_embedding): + nn.init.uniform_(parameter, -self.embedding_range.item(), self.embedding_range.item()) + + def angle(self, value): + parameter = self.embedding_range + token = None if parameter.is_inference() else (id(parameter), parameter._version, parameter.device, parameter.dtype) + if token is None or token != self._range_token: + self._range_value = parameter.item() + self._range_token = token + return (value / self._range_value).tanh() * math.pi + + def embed(self, tree): + axis, aperture = self.embed_batch([tree]) + return axis[0], aperture[0] + + def embed_batch(self, trees): + op = trees[0][0] + if any(tree[0] != op or len(tree) != len(trees[0]) for tree in trees): + raise ValueError('ConE batches must share a query structure') + if op == 'anchor': + axis = self.angle(self.entity_embedding[[tree[1] for tree in trees]]) + return axis, torch.zeros_like(axis) + if op == 'project': + relations = [tree[1] for tree in trees] + return self.cone_proj(*self.embed_batch([tree[2] for tree in trees]), self.angle(self.axis_embedding[relations]), + self.angle(self.arg_embedding[relations])) + if op == 'not': + axis, aperture = self.embed_batch([tree[1] for tree in trees]) + return torch.where(axis >= 0, axis - math.pi, axis + math.pi), math.pi - aperture + if op == 'and': + axes, apertures = zip(*(self.embed_batch([tree[index] for tree in trees]) for index in range(1, len(trees[0])))) + return self.cone_intersection(torch.stack(axes), torch.stack(apertures)) + raise ValueError('ConE unions must be executed as DNF branches') + + def encode_trees(self, trees): + branches = [union_branches(tree) for tree in trees] + if len({len(group) for group in branches}) != 1: + raise ValueError('ConE batches must share a query structure') + axes, apertures = self.embed_batch([branch for group in branches for branch in group]) + return [values.reshape(len(trees), len(branches[0]), -1) for values in (axes, apertures)] + + def score_embeddings(self, embeddings): + return torch.stack([self._score_branches(list(zip(axes, apertures))) for axes, apertures in zip(*embeddings)]) + + def _score_branches(self, branches): + chunks = [] + geometry = [(axis, (aperture / 2).sin().abs(), axis - aperture, axis + aperture) + for axis, aperture in branches] + self.angle_cache.refresh(self) + for index, raw in enumerate(self.entity_embedding.split(self.candidate_batch_size)): + key = (self.candidate_batch_size, index) + entities = self.angle_cache.get(key) if not torch.is_grad_enabled() else None + if entities is None: + entities = self.angle(raw) + if not torch.is_grad_enabled(): + self.angle_cache.put(key, entities, optional=True) + scores = [] + for axis, radius, lower, upper in geometry: + center = ((entities - axis) / 2).sin().abs() + outside = torch.minimum(((entities - lower) / 2).sin().abs(), + ((entities - upper) / 2).sin().abs()) + outside = outside.masked_fill(center < radius, 0) + distance = outside.norm(p=1, dim=-1) + self.cen * torch.minimum(center, radius).norm(p=1, dim=-1) + scores.append(self.gamma - distance * self.modulus) + chunks.append(torch.stack(scores).max(0).values) + return torch.cat(chunks) + + def score_tree(self, tree): + return self.score_embeddings(self.encode_trees([tree]))[0] diff --git a/dicee/query_answering/methods/cqd.py b/dicee/query_answering/methods/cqd.py new file mode 100644 index 000000000..878d1dd4e --- /dev/null +++ b/dicee/query_answering/methods/cqd.py @@ -0,0 +1,191 @@ +"""Released +H CQD semantics with bounded intermediate score matrices.""" + +import torch + +from ._common import QueryMethod, TensorCache, complex_rows, require_complex + + +class CQD(QueryMethod): + def __init__(self, model, context, *, beam_size=64, tnorm='prod', hybrid=False, max_norm=None, + max_k=512, normalize=True, sigmoid=False, reference_quirks=True, row_batch_size=100, cache_bytes=64 * 2**20, + reference_batching=False, atomic_negation=False, final_batch_size=None): + super().__init__(context) + require_complex(model, context) + if min(beam_size, row_batch_size) < 1 or (max_k is not None and max_k < 1): + raise ValueError('Beam and batch sizes must be positive') + if tnorm not in ('prod', 'min'): + raise ValueError('Choose prod or min') + self.model, self.beam_size, self.tnorm = model, beam_size, tnorm + self.max_norm = (.9 if hybrid else 1.) if max_norm is None else max_norm + if not 0 < self.max_norm <= 1 or hybrid != (self.max_norm != 1): + raise ValueError('Hybrid requires max_norm < 1; plain CQD requires max_norm = 1') + self.hybrid, self.max_k = hybrid, max_k + self.normalize, self.sigmoid = normalize, sigmoid + self.reference_quirks, self.row_batch_size = reference_quirks, row_batch_size + self.reference_batching = reference_batching + self.atomic_negation = atomic_negation + if final_batch_size is not None and (type(final_batch_size) is not int or final_batch_size < 1): + raise ValueError('Final projection batch size must be positive') + self.final_batch_size = final_batch_size + self.cache = TensorCache(cache_bytes) + self._tails = ({h: {r: tuple(sorted(ts)) for r, ts in rels.items()} for h, rels in context.outgoing.items()} + if hybrid else {}) + + def conjunction(self, left, right): + return left * right if self.tnorm == 'prod' else torch.minimum(left, right) + + def _raw(self, heads, relation): + self.cache.refresh(self.model) + key = relation, tuple(heads.tolist()) + scores = self.cache.get(key) + if scores is None: + if heads.is_cuda: + # Keep reference matrix geometry; fail before a predictable OOM. + element = self.model.entity_embeddings.weight.element_size() + dim = self.model.entity_embeddings.weight.shape[1] + scratch = len(heads) * (3 * self.context.num_entities + 6 * dim) * element + free, _ = torch.cuda.mem_get_info(heads.device) + reusable = torch.cuda.memory_reserved(heads.device) - torch.cuda.memory_allocated(heads.device) + if scratch + 256 * 2**20 > free + reusable: + raise MemoryError( + f'CQD reference stage with {len(heads)} heads needs approximately ' + f'{scratch / 2**30:.2f} GiB scratch plus 0.25 GiB reserve; ' + f'{(free + reusable) / 2**30:.2f} GiB is available. ' + 'Use more VRAM or explicitly re-verify a bounded batching configuration.') + scores = complex_rows(self.model, heads, torch.full_like(heads, relation)) + self.cache.put(key, scores) + return scores.sigmoid() * self.max_norm if self.sigmoid else scores + + def _rows(self, heads, relation, observed_relation=None, batch_size=None): + # The release normalizes the entire stage, rather than each atomic row. + low, high = None, None + batch_size = batch_size or (len(heads) if self.reference_batching else self.row_batch_size) + single = len(heads) <= batch_size + raw = None + if self.normalize: + for chunk in heads.split(batch_size): + raw = self._raw(chunk, relation) + lo, hi = raw.min(), raw.max() + low = lo if low is None else torch.minimum(low, lo) + high = hi if high is None else torch.maximum(high, hi) + if not single: + raw = None + if high == low: + raise ValueError('Released CQD min-max normalization is undefined for constant scores') + for start in range(0, len(heads), batch_size): + chunk = heads[start:start + batch_size] + scores = raw if raw is not None else self._raw(chunk, relation) + raw = None + if self.normalize: + scores = self.max_norm * (scores - low) / (high - low) + if self.hybrid: + if not self.normalize: + scores = scores.clone() + rel = relation if observed_relation is None else observed_relation + self._restore_facts(scores, chunk, rel) + yield start, scores + + def _restore_facts(self, scores, heads, relation): + rows, tails = [], [] + for row, head in enumerate(heads.tolist()): + facts = self._tails.get(head, {}).get(relation, ()) + for start in range(0, len(facts), 65536): + part = facts[start:start + 65536] + if len(rows) + len(part) > 65536: + scores[rows, tails] = 1 + rows, tails = [], [] + rows.extend([row] * len(part)) + tails.extend(part) + if rows: + scores[rows, tails] = 1 + + def _width(self, observed): + width = min(self.beam_size + (observed if self.hybrid else 0), self.context.num_entities) + return min(width, self.max_k) if self.hybrid and self.max_k is not None else width + + def _select_atomic(self, heads, relation): + widths = ([self._width(len(self._tails.get(h, {}).get(relation, ()))) for h in heads.tolist()] + if self.hybrid else [self._width(0)] * len(heads)) + width = max(widths) + values, indices = [], [] + for start, rows in self._rows(heads, relation): + if self.reference_batching and not self.hybrid: + return rows.topk(width, dim=1) + for offset, row in enumerate(rows): + count = widths[start + offset] + score, index = row.unsqueeze(0).topk(count, dim=1) + score, index = score[0], index[0] + # The upstream hybrid pads shorter branches with entity 0 / score 0. + values.append(torch.nn.functional.pad(score, (0, width - count))) + indices.append(torch.nn.functional.pad(index, (0, width - count))) + return torch.stack(values), torch.stack(indices) + + def _last_projection(self, heads, relation, prefix, observed_relation=None, negate=False, batch_size=None): + result = None + for start, rows in self._rows(heads, relation, observed_relation, batch_size): + if negate: + rows = 1 - rows + scores = self.conjunction(prefix[start:start + len(rows), None], rows).max(0).values + result = scores if result is None else torch.maximum(result, scores) + return result + + def _path(self, head, relations, negate_last=False): + heads = torch.tensor([head], device=self.device) + if len(relations) == 1: + scores = next(self._rows(heads, relations[0]))[1][0] + return 1 - scores if negate_last else scores + scores = [] + for relation in relations[:-1]: + values, tails = self._select_atomic(heads, relation) + scores.append(values) + heads = tails.flatten() + last_relation = relations[-1] + if self.reference_quirks and len(relations) == 4: + prefix, last_relation = scores[-1].flatten(), relations[-2] + else: + prefix = scores[0].flatten() + for index, values in enumerate(scores[1:], 1): + expanded = (prefix.repeat(values.shape[1]) if self.reference_quirks and len(relations) == 3 and index == 1 + else prefix.repeat_interleave(values.shape[1])) + prefix = self.conjunction(expanded, values.flatten()) + del expanded + del scores + return self._last_projection(heads, last_relation, prefix, relations[-1], negate=negate_last, + batch_size=self.final_batch_size if len(relations) == 4 else None) + + def score_tree(self, tree): + op = tree[0] + if op == 'not': + if not self.atomic_negation: + raise ValueError('CQD negated queries require atomic_negation=true') + child, relations = tree[1], [] + while child[0] == 'project': + relations.append(child[1]) + child = child[2] + if child[0] != 'anchor' or not relations or len(relations) > 2: + raise ValueError('Atomic negation supports anchored one- and two-edge paths') + return self._path(child[1], list(reversed(relations)), negate_last=True) + if op in ('and', 'or'): + values = [self.score_tree(child) for child in tree[1:]] + result = values[0] + for value in values[1:]: + result = (self.conjunction(result, value) if op == 'and' else + 1 - (1 - result) * (1 - value) if self.tnorm == 'prod' else torch.maximum(result, value)) + return result + if op != 'project': + raise ValueError('CQD needs anchored relation projections') + relations, child = [], tree + while child[0] == 'project': + relations.append(child[1]) + child = child[2] + relations.reverse() + if child[0] == 'anchor': + return self._path(child[1], relations) + if len(relations) != 1: + raise ValueError('Only published CQD structures are supported') + values = self.score_tree(child) + count = self._width(int((values == 1).sum()) if self.hybrid else 0) + if self.max_k is not None: + count = min(count, self.max_k) + prefix, heads = values.topk(count) + return self._last_projection(heads, relations[0], prefix) diff --git a/dicee/query_answering/methods/gnnqe.py b/dicee/query_answering/methods/gnnqe.py new file mode 100644 index 000000000..d5b94ca6b --- /dev/null +++ b/dicee/query_answering/methods/gnnqe.py @@ -0,0 +1,119 @@ +"""GNN-QE's NBFNet inference without TorchDrug runtime dependencies.""" + +import torch +from torch import nn + +from ._common import FuzzyGNN +from ._ordered_aggregation import aggregate, cached_layout + + +def degree_statistics(degree, pna): + """Prepare graph-only values on their original device without reassociation.""" + count = degree[:, None, None] + 1 + scales = None + if pna: + scale = count.log() + scale = scale / scale.mean() + scales = torch.cat((torch.ones_like(scale), scale, 1 / scale.clamp_min(1e-2)), -1) + return count, scales + + +class GNNQELayer(nn.Module): + def __init__(self, dim, num_relations, *, aggregate='pna', layer_norm=True, dependent=True): + super().__init__() + if aggregate not in ('pna', 'sum', 'mean', 'max'): + raise ValueError('Unsupported GNN-QE aggregation') + self.dim, self.num_relations, self.aggregate = dim, num_relations, aggregate + self.linear = nn.Linear(dim * (13 if aggregate == 'pna' else 2), dim) + self.layer_norm = nn.LayerNorm(dim) if layer_norm else None + if dependent: + self.relation_linear = nn.Linear(dim, num_relations * dim) + else: + self.relation = nn.Embedding(num_relations, dim) + + def forward(self, hidden, boundary, query, edges, degree, statistics=None): + relations = (self.relation_linear(query).reshape(len(query), self.num_relations, self.dim).transpose(0, 1) + if hasattr(self, 'relation_linear') else self.relation.weight[:, None].expand(-1, len(query), -1)) + if self.aggregate == 'pna': + total, squares, high, low = aggregate(hidden, relations, edges, 'pna') + else: + total = aggregate(hidden, relations, edges) if self.aggregate != 'max' else None + high = aggregate(hidden, relations, edges, 'max') if self.aggregate == 'max' else None + + if self.aggregate in ('mean', 'pna'): + count, scales = statistics if statistics is not None else degree_statistics(degree, self.aggregate == 'pna') + if self.aggregate == 'sum': + update = total + boundary + elif self.aggregate == 'mean': + update = (total + boundary) / count + elif self.aggregate == 'max': + update = torch.maximum(high, boundary) + else: + mean = (total + boundary) / count + variance = (squares + boundary.square()) / count - mean.square() + features = torch.stack((mean, torch.maximum(high, boundary), torch.minimum(low, boundary), + variance.clamp_min(1e-6).sqrt()), -1).flatten(-2) + update = (features[..., None] * scales[:, :, None]).flatten(-2) + value = self.linear(torch.cat((hidden, update), -1)) + if self.layer_norm is not None: + value = self.layer_norm(value) + return value.relu() + + +class GNNQEMLP(nn.Module): + def __init__(self, dim, num_layers): + super().__init__() + self.layers = nn.ModuleList([nn.Linear(dim, dim if index + 1 < num_layers else 1) for index in range(num_layers)]) + + def forward(self, value): + for index, layer in enumerate(self.layers): + value = layer(value) + if index + 1 < len(self.layers): + value = value.relu() + return value + + +class GNNQE(FuzzyGNN): + """Full-graph inference with ordered sparse reductions. + + edge_batch_size and candidate_batch_size describe manifest metadata; + they do not chunk execution or bound memory. Query batch size controls the + dense state size. + """ + def __init__(self, context, *, dim=32, num_layers=4, aggregate='pna', short_cut=True, + layer_norm=True, dependent=True, concat_hidden=False, mlp_layers=2, logic='product', + edge_batch_size=65536, candidate_batch_size=4096): + super().__init__(context, logic) + if min(dim, num_layers, mlp_layers, edge_batch_size, candidate_batch_size) < 1: + raise ValueError('GNN-QE dimensions and batch sizes must be positive') + self.short_cut, self.concat_hidden = short_cut, concat_hidden + self.edge_batch_size, self.candidate_batch_size = edge_batch_size, candidate_batch_size + self.query = nn.Embedding(context.num_relations, dim) + self.layers = nn.ModuleList([GNNQELayer(dim, context.num_relations, aggregate=aggregate, + layer_norm=layer_norm, dependent=dependent) for _ in range(num_layers)]) + self.mlp = GNNQEMLP(dim * (num_layers + 1 if concat_hidden else 2), mlp_layers) + triples = torch.tensor(context.triples, dtype=torch.long).reshape(-1, 3) + self.register_buffer('edge_index', triples[:, [2, 0]].T, persistent=False) + self.register_buffer('edge_type', triples[:, 1], persistent=False) + self.register_buffer('degree', torch.bincount(triples[:, 2], minlength=context.num_entities).float(), persistent=False) + self._layouts = {} + + def project(self, membership, relation): + return self.project_batch(membership[None], torch.tensor([relation], device=self.device))[0] + + def project_batch(self, membership, relation, *, relation_ids=None): + edges = cached_layout(self._layouts, 'entity', self.edge_index, self.edge_type, + self.context.num_entities, relation_order=True) + query = self.query(relation) + boundary = torch.einsum('bn,bd->nbd', membership, query) + hidden, hiddens = boundary, [] + statistics = (degree_statistics(self.degree, any(layer.aggregate == 'pna' for layer in self.layers)) + if any(layer.aggregate in ('mean', 'pna') for layer in self.layers) else None) + for layer in self.layers: + update = layer(hidden, boundary, query, edges, self.degree, statistics) + hidden = update + hidden if self.short_cut else update + if self.concat_hidden: + hiddens.append(hidden) + values = hiddens if self.concat_hidden else [hidden] + features = torch.cat([*values, query.expand(self.context.num_entities, -1, -1)], -1) + return self.mlp(features).squeeze(-1).sigmoid().t() diff --git a/dicee/query_answering/methods/heuristic.py b/dicee/query_answering/methods/heuristic.py new file mode 100644 index 000000000..e9119d9be --- /dev/null +++ b/dicee/query_answering/methods/heuristic.py @@ -0,0 +1,58 @@ +"""Released incoming-relation heuristic with reproducible random rankings.""" + +import hashlib +import json + +import torch +from torch import nn + +from .._query import compile_query, validate_tree +from ._common import FuzzyGNN + + +def query_seed(query, seed): + def flatten(value): + return [i for part in value for i in flatten(part)] if isinstance(value, tuple) else [value] + payload = json.dumps([seed, flatten(query)], separators=(',', ':')).encode() + return int.from_bytes(hashlib.sha256(payload).digest()[:8], 'little') % (2**63) + + +class IncomingRelationHeuristic(FuzzyGNN): + """Ignore projection heads; use Boolean logic and shuffle within two classes. + + Each query has its own seeded random stream, making subsets, batching and + interrupted/resumed runs reproduce the same ranking. + """ + + def __init__(self, context, *, seed=0): + super().__init__(context, logic='godel') + if type(seed) is not int: + raise ValueError('Heuristic seed must be an integer') + self.seed = seed + self.dummy_param = nn.Parameter(torch.zeros(1)) + triples = torch.tensor(context.triples, dtype=torch.long).reshape(-1, 3) + self.register_buffer('relations', triples[:, 1], persistent=False) + self.register_buffer('tails', triples[:, 2], persistent=False) + + def project_batch(self, membership, relation, *, relation_ids=None): + result = membership.new_zeros(membership.shape) + for i, r in enumerate(relation): + result[i, self.tails[self.relations == r]] = 1 + return result + + @torch.no_grad() + def predict_batch(self, queries): + if not queries: + return self.dummy_param.new_empty((0, self.context.num_entities)) + trees = [compile_query(q) for q in queries] + for tree in trees: + validate_tree(tree, self.context.num_entities, self.context.num_relations) + probability = self.membership_batch(trees) + noise = torch.stack([torch.rand(self.context.num_entities, device=self.device, + generator=torch.Generator(device=self.device).manual_seed(query_seed(q, self.seed))) + for q in queries]) + probability = (2 * probability + noise) / 3 + return ((probability + 1e-10) / (1 - probability + 1e-10)).log() + + def predict(self, query): + return self.predict_batch([query])[0] diff --git a/dicee/query_answering/methods/qto.py b/dicee/query_answering/methods/qto.py new file mode 100644 index 000000000..806edaa93 --- /dev/null +++ b/dicee/query_answering/methods/qto.py @@ -0,0 +1,203 @@ +"""QTO's calibrated neural adjacency and max-product query execution.""" + +from collections import OrderedDict + +import torch + +from ._common import QueryMethod, TensorCache, cache_token, complex_rows, require_complex + +# Rows times entities densified at once when a projection reads a relation matrix. +DENSE_ELEMENTS = 2**24 +# Projections from at most this many heads compute their rows unless the relation matrix is already on the device. +ROW_HEADS = 8 + + +def _size(matrix): + return sum(tensor.numel() * tensor.element_size() for tensor in matrix) + + +class RelationMatrices: + """Thresholded calibrated relation matrices in CSR form, least recently used out first. + + Matrices stay on the model's device up to ``device_bytes``; evicted ones move to host memory, up to + ``host_bytes``, and return on their next use. Parameter changes invalidate both, as in ``TensorCache``. + """ + + def __init__(self, device_bytes, host_bytes): + if any(type(capacity) is not int or capacity < 0 for capacity in (device_bytes, host_bytes)): + raise ValueError('Matrix cache capacities must be nonnegative byte counts') + self.capacity = dict(device=device_bytes, host=host_bytes) + self.tiers = dict(device=OrderedDict(), host=OrderedDict()) + self.bytes, self.token = dict(device=0, host=0), None + + def refresh(self, model): + token = cache_token(model) + if token is None or token != self.token: + for tier in self.tiers.values(): + tier.clear() + self.bytes, self.token = dict(device=0, host=0), token + + def resident(self, relation): + """The matrix if it is on the device; host memory is not consulted.""" + if relation in self.tiers['device']: + self.tiers['device'].move_to_end(relation) + return self.tiers['device'][relation] + return None + + def get(self, relation, device): + matrix = self.resident(relation) + if matrix is not None: + return matrix + matrix = self.tiers['host'].pop(relation, None) + if matrix is None: + return None + self.bytes['host'] -= _size(matrix) + matrix = tuple(tensor.to(device) for tensor in matrix) + self.put(relation, matrix) + return matrix + + def put(self, relation, matrix): + self._store('device', relation, matrix) + + def _store(self, tier, relation, matrix): + if self.token is None: + return + size = _size(matrix) + if size > self.capacity[tier]: + self._spill(tier, relation, matrix) + return + values = self.tiers[tier] + while self.bytes[tier] + size > self.capacity[tier]: + victim, old = values.popitem(last=False) + self.bytes[tier] -= _size(old) + self._spill(tier, victim, old) + values[relation] = matrix + self.bytes[tier] += size + + def _spill(self, tier, relation, matrix): + if tier == 'device' and matrix[0].device.type != 'cpu': + self._store('host', relation, tuple(tensor.cpu() for tensor in matrix)) + + +class QTO(QueryMethod): + """QTO with the released canonical 100-head calibration. + + With ``matrix_bytes`` > 0, projections read each relation's thresholded calibrated matrix, built once from + the same canonical blocks and kept in CSR form (see ``RelationMatrices``), instead of recomputing rows per + query. Scores are identical; only time and memory change. + """ + + def __init__(self, model, context, *, threshold, negation_scale, row_batch_size=100, cache_bytes=512 * 2**20, + reference_batching=False, matrix_bytes=0, host_matrix_bytes=0): + super().__init__(context) + require_complex(model, context) + if not 0 <= threshold <= 1 or negation_scale <= 0 or row_batch_size < 1: + raise ValueError('Invalid QTO calibration or batch size') + if (matrix_bytes or host_matrix_bytes) and not (reference_batching and matrix_bytes): + raise ValueError('Relation matrices need reference_batching and matrix_bytes > 0') + self.model, self.threshold, self.negation_scale = model, threshold, negation_scale + self.row_batch_size = row_batch_size + self.reference_batching = reference_batching + self.cache = TensorCache(cache_bytes) + self.matrices = RelationMatrices(matrix_bytes, host_matrix_bytes) if matrix_bytes else None + + def calibrated(self, group, relation): + """Calibrated rows of the heads in ``group``, scored together as one block.""" + raw = complex_rows(self.model, group, [relation] * len(group)) + observed = [self.context.outgoing.get(head, {}).get(relation, ()) for head in group] + degrees = raw.new_tensor([max(1, len(tails)) for tails in observed]) + calibrated = raw.softmax(-1) * degrees[:, None] + calibrated = calibrated * (calibrated >= self.threshold) + calibrated = torch.where(calibrated >= 1, .9999, calibrated) + rows = [i for i, tails in enumerate(observed) for _ in tails] + if rows: + calibrated[rows, [tail for tails in observed for tail in tails]] = 1 + return calibrated + + def rows(self, heads, relation): + self.cache.refresh(self.model) + requested = heads.tolist() + values = {head: self.cache.get((head, relation)) for head in requested} + missing = [head for head, value in values.items() if value is None] + groups = ([list(range(start, min(start + 100, self.context.num_entities))) for start in sorted({head // 100 * 100 for head in missing})] + if self.reference_batching else [missing] if missing else []) + for group in groups: + for head, row in zip(group, self.calibrated(group, relation)): + if head in values: + values[head] = row.clone() if self.reference_batching else row + self.cache.put((head, relation), row) + # Keep requested rows ahead of opportunistic canonical-block prefetches. + if groups: + for head, row in values.items(): + if self.cache.get((head, relation)) is None: + self.cache.put((head, relation), row) + return torch.stack([values[head] for head in requested]) + + def matrix(self, relation): + """``relation``'s calibrated matrix as CSR (row pointers, int32 columns, values), from canonical blocks.""" + self.matrices.refresh(self.model) + matrix = self.matrices.get(relation, self.device) + if matrix is None: + n = self.context.num_entities + counts, columns, values = [], [], [] + for start in range(0, n, 100): + block = self.calibrated(list(range(start, min(start + 100, n))), relation) + rows, cols = block.nonzero(as_tuple=True) + counts.append(torch.bincount(rows, minlength=len(block))) + columns.append(cols.to(torch.int32)) + values.append(block[rows, cols]) + pointers = torch.cat([counts[0].new_zeros(1), torch.cat(counts).cumsum(0)]) + matrix = pointers, torch.cat(columns), torch.cat(values) + self.matrices.put(relation, matrix) + return matrix + + def matrix_rows(self, matrix, heads): + """Dense calibrated rows of ``heads`` from a CSR matrix.""" + pointers, columns, values = matrix + starts = pointers[heads] + lengths = pointers[heads + 1] - starts + rows = values.new_zeros(len(heads), self.context.num_entities) + total = int(lengths.sum()) + if total: + local = torch.repeat_interleave(torch.arange(len(heads), device=heads.device), lengths, output_size=total) + offsets = torch.arange(total, device=heads.device) - (lengths.cumsum(0) - lengths)[local] + starts[local] + rows[local, columns[offsets].long()] = values[offsets] + return rows + + def projection(self, prefix, relation, negate=False): + result = torch.zeros_like(prefix) + heads = prefix.nonzero().flatten() + if not len(heads): + return result + matrix = None + if self.matrices is not None: + # Both paths give the same rows; a few heads rarely justify loading or building a whole matrix. + self.matrices.refresh(self.model) + matrix = self.matrices.resident(relation) + if matrix is None and len(heads) > ROW_HEADS: + matrix = self.matrix(relation) + if matrix is None: + chunks = ((chunk, self.rows(chunk, relation)) for chunk in heads.split(self.row_batch_size)) + else: + size = max(1, DENSE_ELEMENTS // self.context.num_entities) + chunks = ((chunk, self.matrix_rows(matrix, chunk)) for chunk in heads.split(size)) + # The maximum over heads does not depend on how they are chunked. + for chunk, rows in chunks: + if negate: + rows = 1 - torch.minimum(torch.ones_like(rows), self.negation_scale * rows) + result = torch.maximum(result, (prefix[chunk, None] * rows).max(0).values) + return result + + def score_tree(self, tree): + op = tree[0] + if op == 'anchor': + return torch.nn.functional.one_hot(torch.tensor(tree[1], device=self.device), self.context.num_entities).float() + if op == 'project': + return self.projection(self.score_tree(tree[2]), tree[1]) + if op == 'not': + child = tree[1] + if child[0] != 'project': + raise ValueError('Official QTO supports negation of relation atoms only') + return self.projection(self.score_tree(child[2]), child[1], negate=True) + values = torch.stack([self.score_tree(child) for child in tree[1:]]) + return values.prod(0) if op == 'and' else 1 - (1 - values).prod(0) diff --git a/dicee/query_answering/methods/ultraquery.py b/dicee/query_answering/methods/ultraquery.py new file mode 100644 index 000000000..414e1e634 --- /dev/null +++ b/dicee/query_answering/methods/ultraquery.py @@ -0,0 +1,70 @@ +"""Native UltraQuery using DICE's checkpoint-compatible ULTRA layers.""" + +import torch + +from ...models.ultra import EntityNBFNet, RelNBFNet, build_relation_graph +from ._common import FuzzyGNN, TensorCache +from ._ordered_aggregation import aggregate, cached_layout + + +class UltraQuery(FuzzyGNN): + def __init__(self, context, *, dim=64, num_layers=6, threshold=0., logic='product', cache_bytes=64 * 2**20, + observed_traversal=False, calibration=None): + super().__init__(context, logic, observed_traversal, calibration) + if not 0 <= threshold <= 1: + raise ValueError('Threshold must be between zero and one') + if calibration and threshold: + raise ValueError('A calibrated projection takes no membership threshold') + self.threshold = threshold + self.relation_model = RelNBFNet(dim, num_layers) + self.entity_model = EntityNBFNet(dim, num_layers) + triples = torch.tensor(context.triples, dtype=torch.long).reshape(-1, 3) + edges, types = triples[:, [0, 2]].T, triples[:, 1] + redges, rtypes = build_relation_graph(edges, types, context.num_entities, context.num_relations) + for name, value in (('edge_index', edges), ('edge_type', types), ('rel_edge_index', redges), ('rel_edge_type', rtypes)): + self.register_buffer(name, value, persistent=False) + self.cache = TensorCache(cache_bytes) + self._layouts = {} + + def project(self, membership, relation): + return self.project_batch(membership[None], torch.tensor([relation], device=self.device), relation_ids=(relation,))[0] + + @staticmethod + def _layer(layer, hidden, boundary, relations, edges): + update = aggregate(hidden.transpose(0, 1), relations.transpose(0, 1), edges).transpose(0, 1) + boundary + value = layer.linear(torch.cat((hidden, update), -1)) + return layer.layer_norm(value).relu() + hidden + + def project_batch(self, membership, relation, *, relation_ids=None, logits=False): + rel_edges = cached_layout(self._layouts, 'relation', self.rel_edge_index, self.rel_edge_type, + self.context.num_relations) + ent_edges = cached_layout(self._layouts, 'entity', self.edge_index, self.edge_type, self.context.num_entities) + self.cache.refresh(self) + key = tuple(relation.tolist()) if relation_ids is None else tuple(relation_ids) + relations = self.cache.get(key) + if relations is None: + boundary = membership.new_zeros(len(relation), self.context.num_relations, self.relation_model.dim) + boundary[torch.arange(len(relation), device=self.device), relation] = 1 + relations = boundary + for layer in self.relation_model.layers: + rel = layer.relation.weight.expand(len(relation), -1, -1) + relations = self._layer(layer, relations, boundary, rel, rel_edges) + self.cache.put(key, relations) + query = relations[torch.arange(len(relation), device=self.device), relation] + if self.threshold > 0: + membership = membership.masked_fill(membership <= self.threshold, 0) + boundary = torch.einsum('bn,bd->bnd', membership, query) + hidden = boundary + for index, layer in enumerate(self.entity_model.layers): + projection_key = ('projection', key, index) + projected = self.cache.get(projection_key) + if projected is None: + projected = layer.relation_projection(relations) + # Optional projections use spare capacity; reasoner outputs have priority. + if key in self.cache.values: + self.cache.put(projection_key, projected, optional=True) + hidden = self._layer(layer, hidden, boundary, projected, ent_edges) + # Preserve the reference MLP arithmetic, including saturated probability ties. + features = torch.cat((hidden, query[:, None].expand_as(hidden)), -1) + output = self.entity_model.mlp(features).squeeze(-1) + return output if logits else output.sigmoid() diff --git a/dicee/query_answering/score_adapter.py b/dicee/query_answering/score_adapter.py new file mode 100644 index 000000000..a3d224758 --- /dev/null +++ b/dicee/query_answering/score_adapter.py @@ -0,0 +1,313 @@ +"""Small monotone row transforms shared by adapter fitting and query inference. + +Memberships are fuzzy scores, not a claim of probability calibration. +""" + +import hashlib +import json +import math +import weakref +from pathlib import Path +from typing import Any + +import torch +from torch import nn +from torch.nn import functional as F + +from ._deferred import require +from .context import checkpoint_state, state_fingerprint, state_token + +FEATURE_COUNTS = {'global': 1, 'context': 4, 'context_scores': 8} +# Fixed calibrations of published executors, applied instead of the sigmoid when ``fixed_calibration`` selects them: +# CQD-Hybrid's min-max normalization below the observed-fact membership, and QTO's softmax times the observed degree. +FIXED_CALIBRATIONS = ('minmax', 'softmax-degree', 'softmax', 'softmax-degree-ties', 'softmax-mass') +# Softmax calibrations: QTO's softmax times the observed degree, the plain softmax (degree one), QTO with the ties at +# its cap ordered by the raw score, so that a beam or a ranking keeps the backbone's order among capped entities, and +# FIT's softmax times the observed degree over the softmax mass on the observed tails (softmax-mass). +SOFTMAX_CALIBRATIONS = ('softmax-degree', 'softmax', 'softmax-degree-ties', 'softmax-mass') +MINMAX_CEILING = .9 +SOFTMAX_DEGREE_CEILING = .9999 +# softmax-degree-ties lowers a capped log-membership by CAP_TIE_EPSILON * (1/2 - atan(raw) / pi), less than this +# much and strictly decreasing in the raw score: resolvable at the cap and after summing log-memberships over atoms. +CAP_TIE_EPSILON = 1e-6 +# With observed_tie_break, a known fact gets membership exp(-OBSERVED_TIE_EPSILON * (1 - mu)), mu its calibrated +# membership, instead of exactly one: still one for every practical purpose, but a beam over more known tails than it +# holds keeps those the backbone scores highest instead of those with the lowest entity IDs. +OBSERVED_TIE_EPSILON = 1e-9 + + +def canonical_feature_mode(mode): + """Resolve supported feature-mode names to their canonical spelling.""" + mode = 'context_scores' if mode == 'context_scores_v1' else mode + if mode not in FEATURE_COUNTS: + raise ValueError(f'Unknown feature mode: {mode}') + return mode + + +def score_features(raw, observed, base, mode): + mode = canonical_feature_mode(mode) + raw = raw.to(torch.float64) + if raw.ndim != 2 or not raw.shape[1] or observed.shape != raw.shape or base.shape != (len(raw), 4): + raise ValueError('Features require complete [rows, entities] scores and four context features') + require(torch.isfinite(raw).all() & torch.isfinite(base).all(), ValueError('Atomic rows and features must be finite')) + if mode == 'global': + return base[:, :1] + if mode == 'context': + return base + n = raw.shape[1] + mean = raw.mean(1) + centered = raw - raw.max(1, keepdim=True).values + exp = centered.exp() + normalizer = exp.sum(1) + entropy = normalizer.log() - (exp * centered).sum(1) / normalizer + if n > 1: + entropy = entropy / math.log(n) + top = raw.topk(2, dim=1).values + gap = ((top[:, 0] - top[:, 1]) / 4).tanh() + else: + entropy, gap = torch.zeros_like(mean), torch.zeros_like(mean) + count = observed.sum(1) + obs_mean = (raw * observed).sum(1) / count.clamp_min(1) + contrast = torch.where(count > 0, ((obs_mean - mean) / 4).tanh(), 0.) + return torch.cat((base, torch.stack(((mean / 4).tanh(), entropy, gap, contrast), dim=1)), dim=1) + + +class QueryScoreAdapter(nn.Module): + """Positive row scale and bias; zero weights give the sigmoid baseline. + + ``global`` learns only two parameters. ``context`` and ``context_scores`` + learn eight and sixteen with a linear head. Context observations can override or + interpolate memberships. Newly fitted adapters bind to a backbone state hash. + ``membership_threshold`` zeroes memberships at or below a fixed value, the hand-set + calibration of UltraQuery LP; it is not trained. ``fixed_calibration`` replaces the sigmoid by a training-free + row calibration: ``minmax`` (CQD-Hybrid: 0.9 times the min-max normalized row), ``softmax-degree`` (QTO: the + softmax over tails times the number of observed tails, at least one, capped below one) or ``softmax-mass`` (FIT: the + softmax times the number of observed tails over their softmax mass, capped below one). They take no weights. + """ + + def __init__(self, feature_mode='context_scores', observed_mix=0., *, weights=None, metadata=None, + bias_bound=4., scale_bound=2., normalization='none', hidden_dim=0, hidden_weights=None, seed=0, + membership_threshold=0., fixed_calibration=None, mask_known_logits=False): + super().__init__() + feature_mode = canonical_feature_mode(feature_mode) + if not math.isfinite(observed_mix) or not 0 <= observed_mix <= 1: + raise ValueError('observed_mix must be in [0, 1]') + if not math.isfinite(bias_bound) or bias_bound <= 0: + raise ValueError('bias_bound must be finite and positive') + if scale_bound is not None and (not math.isfinite(scale_bound) or scale_bound <= 1): + raise ValueError('scale_bound must exceed one, or be None for unrestricted scaling') + if normalization not in ('none', 'standard'): + raise ValueError('Choose none or standard row normalization') + if not math.isfinite(membership_threshold) or not 0 <= membership_threshold < 1: + raise ValueError('membership_threshold must be in [0, 1)') + if type(hidden_dim) is not int or hidden_dim < 0: + raise ValueError('hidden_dim must be a nonnegative integer') + if fixed_calibration not in (None, *FIXED_CALIBRATIONS): + raise ValueError(f'Choose a fixed calibration: {", ".join(FIXED_CALIBRATIONS)}, or None for the sigmoid') + if fixed_calibration and (feature_mode != 'global' or hidden_dim or normalization != 'none' + or (weights is not None and bool(torch.as_tensor(weights).any()))): + raise ValueError('Training-free calibrations take no weights, features or normalization') + if mask_known_logits and fixed_calibration not in SOFTMAX_CALIBRATIONS: + raise ValueError('Masking known-tail logits applies to the softmax calibrations only') + if mask_known_logits and fixed_calibration == 'softmax-mass': + raise ValueError('softmax-mass divides by the softmax mass on the known tails, so they cannot leave the softmax') + self.feature_mode, self.observed_mix = feature_mode, float(observed_mix) + self.bias_bound, self.normalization, self.hidden_dim = float(bias_bound), normalization, hidden_dim + self.scale_bound = None if scale_bound is None else float(scale_bound) + self.membership_threshold = float(membership_threshold) + self.fixed_calibration = fixed_calibration + self.mask_known_logits = bool(mask_known_logits) + self.observed_tie_break = False # an evaluation option of the recipe, not part of the adapter artifact + features = FEATURE_COUNTS[feature_mode] + size = (2, hidden_dim or features) + value = torch.zeros(size, dtype=torch.float64) if weights is None else torch.as_tensor(weights, dtype=torch.float64) + if value.shape != size or not torch.isfinite(value).all(): + raise ValueError(f'Expected finite adapter weights of shape {size}') + self.weights = nn.Parameter(value.clone()) + if hidden_dim: + value = (torch.randn(hidden_dim, features, dtype=torch.float64, + generator=torch.Generator().manual_seed(seed)) / math.sqrt(features) + if hidden_weights is None else torch.as_tensor(hidden_weights, dtype=torch.float64)) + if value.shape != (hidden_dim, features) or not torch.isfinite(value).all(): + raise ValueError('Invalid hidden adapter weights') + self.hidden_weights = nn.Parameter(value.clone()) + else: + if hidden_weights is not None: + raise ValueError('Hidden weights require hidden_dim') + self.register_parameter('hidden_weights', None) + self.metadata = dict(metadata or {}) + + @property + def configuration(self): + configuration = dict(feature_mode=self.feature_mode, observed_mix=self.observed_mix, bias_bound=self.bias_bound, + scale_bound=self.scale_bound, normalization=self.normalization, hidden_dim=self.hidden_dim) + if self.membership_threshold: + configuration['membership_threshold'] = self.membership_threshold + if self.fixed_calibration: + configuration['fixed_calibration'] = self.fixed_calibration + if self.mask_known_logits: + configuration['mask_known_logits'] = True + return configuration + + @property + def needs_context(self): + """Whether rows need the observed facts of a context graph (features, overrides or observed degrees).""" + return self.feature_mode != 'global' or bool(self.observed_mix) or self.fixed_calibration in SOFTMAX_CALIBRATIONS + + def prepare(self, raw, observed=None, base=None): + """Prepare parameter-independent inputs once for frozen training banks.""" + raw = raw.to(dtype=torch.float64) + if raw.ndim != 2: + raise ValueError('Adapter requires [rows, entities] logits') + if observed is None or base is None: + if self.needs_context: + raise ValueError('Context features/observations require an explicit context graph') + observed = torch.zeros_like(raw, dtype=torch.bool) + base = raw.new_zeros((len(raw), 4)) + base[:, 0] = 1 + features = score_features(raw, observed, base, self.feature_mode) + if self.normalization == 'standard': + raw = (raw - raw.mean(1, keepdim=True)) / raw.std(1, correction=0, keepdim=True).clamp_min(1e-6) + return raw, observed, features + + def _on(self, name, device): + """Parameter on ``device``; inference reuses one copy until the parameter changes.""" + value = getattr(self, name) + if value.device == device or (torch.is_grad_enabled() and value.requires_grad) or value.is_inference(): + return value.to(device=device) + key = (device, id(value), value._version) + copies = self.__dict__.setdefault('_device_copies', {}) + if copies.get(name, (None,))[0] != key: + copies[name] = key, value.detach().to(device=device) + return copies[name][1] + + def transform(self, raw, observed, features): + if self.fixed_calibration: + return self._fixed(raw, observed) + weights = self._on('weights', raw.device) + if self.hidden_weights is not None: + features = (features @ self._on('hidden_weights', raw.device).T).tanh() + u, v = (features @ weights.T).unbind(1) + # Match the derivative at initialization across scale bounds. + log_scale = math.log(2) * u + if self.scale_bound is not None: + limit = math.log(self.scale_bound) + log_scale = limit * (u * (math.log(2) / limit)).tanh() + scale = log_scale.exp() + require(torch.isfinite(scale).all() & ~(scale <= 0).any(), FloatingPointError('Nonfinite or zero adapter scale')) + logits = scale[:, None] * raw + self.bias_bound * v.tanh()[:, None] + return self._finish(F.logsigmoid(logits), observed) + + def _fixed(self, raw, observed): + if self.fixed_calibration == 'minmax': + low, high = raw.min(1, keepdim=True).values, raw.max(1, keepdim=True).values + # A constant row has no order to normalize; all its memberships become zero. + logs = (MINMAX_CEILING * (raw - low) / torch.where(high > low, high - low, 1.)).log() + else: + scores = raw + if self.mask_known_logits: + # Known tails leave the softmax (a row of known tails only keeps its scores); the override still + # gives them membership one. + scores = torch.where(observed.all(1, keepdim=True), raw, raw.masked_fill(observed, -math.inf)) + logs = scores.log_softmax(1) + if self.fixed_calibration == 'softmax-mass': + # FIT (Yin et al., 2024, Section 5.2): the softmax is scaled so that the observed tails together hold + # their count, which makes their mean membership one before the cap. A row without observed tails keeps + # the softmax. The cap stays QTO's 0.9999, below the membership of an observed fact. + count = observed.sum(1, keepdim=True) + mass = logs.masked_fill(~observed, -math.inf).logsumexp(1, keepdim=True) + logs = logs + torch.where(count > 0, count.clamp_min(1).to(raw.dtype).log() - mass, 0.) + elif self.fixed_calibration != 'softmax': + logs = logs + observed.sum(1, keepdim=True).clamp_min(1).to(raw.dtype).log() + capped = logs >= math.log(SOFTMAX_DEGREE_CEILING) + logs = logs.clamp_max(math.log(SOFTMAX_DEGREE_CEILING)) + if self.fixed_calibration == 'softmax-degree-ties': + logs = torch.where(capped, logs - CAP_TIE_EPSILON * (.5 - torch.atan(raw) / math.pi), logs) + return self._finish(logs, observed) + + def _finish(self, logs, observed): + if self.membership_threshold: + # UltraQuery LP's fixed calibration: memberships at or below the threshold become exactly zero. + logs = logs.masked_fill(logs <= math.log(self.membership_threshold), -math.inf) + if self.observed_mix == 1: + if self.observed_tie_break: + logs = torch.where(observed, OBSERVED_TIE_EPSILON * torch.expm1(logs), logs) + else: + logs = logs.masked_fill(observed, 0.) + elif self.observed_mix: + mixed = torch.logaddexp(logs.new_tensor(math.log(self.observed_mix)), math.log1p(-self.observed_mix) + logs) + logs = torch.where(observed, mixed, logs) + return logs + + def forward(self, raw, observed=None, base=None): + return self.transform(*self.prepare(raw, observed, base)) + + def verify_model(self, model): + expected = self.metadata.get('backbone_state_sha256') + if not expected: + return + token = state_token(model) + previous = getattr(self, '_verified_model', None) + if (token is not None and previous is not None and previous() is model + and getattr(self, '_verified_state', None) == (expected, token)): + return + if state_fingerprint(model) != expected: + raise ValueError('Adapter belongs to different or modified backbone weights') + self._verified_model = weakref.ref(model) + self._verified_state = (expected, token) if token is not None else None + + def to_dict(self): + return dict(self.metadata, version=3, **self.configuration, weights=self.weights.detach().cpu().tolist(), + hidden_weights=None if self.hidden_weights is None else self.hidden_weights.detach().cpu().tolist()) + + def save(self, path: str | Path) -> None: + """Write the adapter and its metadata as JSON.""" + Path(path).write_text(json.dumps(self.to_dict(), indent=2, allow_nan=False) + '\n') + + @classmethod + def load(cls, path: str | Path, *, model: Any, key: str | None = None, + checkpoint: str | Path | None = None) -> 'QueryScoreAdapter': + """Load an adapter artifact, or one entry of an adapter catalog. + + Args: + path: Adapter or catalog JSON file. + model: Backbone the adapter was fitted for; its state must match the recorded fingerprint. + key: Catalog entry to load. + checkpoint: Original backbone checkpoint, for adapters that record its file hash; + both its bytes and its tensor state are checked. + + Returns: + The adapter. + + Raises: + ValueError: If the file is not a supported adapter, or the backbone differs. + """ + payload = json.loads(Path(path).read_text()) + if key is not None: + if key not in payload: + raise ValueError(f'Adapter catalog has no entry {key!r}') + payload = payload[key] + if 'weights' not in payload: + raise ValueError('Select a catalog entry with key=...') + if payload.get('version', 1) not in (1, 2, 3): + raise ValueError('Unsupported adapter artifact version') + fields = ('weights', 'feature_mode', 'observed_mix', 'version', 'bias_bound', 'scale_bound', 'normalization', 'hidden_dim', + 'hidden_weights', 'membership_threshold', 'fixed_calibration', 'mask_known_logits') + metadata = {k: v for k, v in payload.items() if k not in fields} + if 'backbone_state_sha256' not in metadata: + if checkpoint is None or 'backbone_checkpoint_sha256' not in metadata: + raise ValueError('Adapter needs verified backbone state or its original checkpoint') + checkpoint = Path(checkpoint) + with checkpoint.open('rb') as stream: + checksum = hashlib.file_digest(stream, 'sha256').hexdigest() + if checksum != metadata['backbone_checkpoint_sha256']: + raise ValueError('Adapter checkpoint checksum mismatch') + metadata['backbone_state_sha256'] = state_fingerprint(checkpoint_state(checkpoint)) + adapter = cls(payload.get('feature_mode', 'context'), payload.get('observed_mix', 0.), + weights=payload['weights'], metadata=metadata, bias_bound=payload.get('bias_bound', 4.), + scale_bound=payload.get('scale_bound', 2.), + normalization=payload.get('normalization', 'none'), hidden_dim=payload.get('hidden_dim', 0), + hidden_weights=payload.get('hidden_weights'), membership_threshold=payload.get('membership_threshold', 0.), + fixed_calibration=payload.get('fixed_calibration'), mask_known_logits=payload.get('mask_known_logits', False)) + adapter.verify_model(model) + return adapter diff --git a/dicee/query_generator.py b/dicee/query_generator.py index 1bbb105bb..cf911f0a2 100644 --- a/dicee/query_generator.py +++ b/dicee/query_generator.py @@ -8,6 +8,7 @@ import numpy as np +from .query_answering._query import QUERY_SHAPES, compile_query, exact_answers from .static_funcs import load_pickle, save_pickle logger = logging.getLogger(__name__) @@ -34,27 +35,28 @@ def __init__(self, train_path, val_path: str, test_path: str, ent2id: Optional[D self.rel2id: Dict = rel2id self.ent_in: Dict = {} self.ent_out: Dict = {} - self.query_name_to_struct = {"1p": ['e', ['r']], - "2p": ['e', ['r', 'r']], - "3p": ['e', ['r', 'r', 'r']], - "2i": [['e', ['r']], ['e', ['r']]], # @TODO: double check the evaluation - "3i": [['e', ['r']], ['e', ['r']], ['e', ['r']]], - "pi": [['e', ['r', 'r']], ['e', ['r']]], - "ip": [[['e', ['r']], ['e', ['r']]], ['r']], - "2in": [['e', ['r']], ['e', ['r', 'n']]], - "3in": [['e', ['r']], ['e', ['r']], ['e', ['r', 'n']]], - "pin": [['e', ['r', 'r']], ['e', ['r', 'n']]], - "pni": [['e', ['r', 'r', 'n']], ['e', ['r']]], - "inp": [[['e', ['r']], ['e', ['r', 'n']]], ['r']], - # union - "2u": [['e', ['r']], ['e', ['r']], ['u']], - "up": [[['e', ['r']], ['e', ['r']], ['u']], ['r']]} + self.query_name_to_struct = {name: self.tuple2list(shape) for name, shape in QUERY_SHAPES.items()} self.set_global_seed(seed) # Sanity checking assert isinstance(self.ent2id, dict) or self.ent2id is None assert isinstance(self.rel2id, dict) or self.rel2id is None + @classmethod + def from_context(cls, context, *, seed=1): + """Ground queries from an in-memory context without files or global RNG changes.""" + instance = cls.__new__(cls) + instance.num_entities = context.num_entities + instance.ent2id = {str(i): i for i in range(context.num_entities)} + instance.rng = random.Random(seed) + instance.inverse_relations = {a: b for h, t in context.inverse_relations for a, b in ((h, t), (t, h))} + instance.ent_in = defaultdict(lambda: defaultdict(set)) + instance.ent_out = defaultdict(lambda: defaultdict(set)) + for h, r, t in context.triples: + instance.ent_in[t][r].add(h) + instance.ent_out[h][r].add(t) + return instance + def list2tuple(self, list_data): # @TODO: add description return tuple(self.list2tuple(x) if isinstance(x, list) else x for x in list_data) @@ -70,6 +72,7 @@ def tuple2list(self, x: Union[List, Tuple]) -> Union[List, Tuple]: def set_global_seed(self, seed: int): """Set seed""" + self.rng = random.Random(seed) np.random.seed(seed) random.seed(seed) @@ -117,15 +120,15 @@ def fill_query(self, query_structure: List[Union[str, List]], for j in range(40): if len(ent_in[answer].keys()) < 1: return True # not enough relations, return True to indicate broken flag - r_tmp = random.sample(list(ent_in[answer].keys()), 1)[0] - if r_tmp // 2 != r // 2 or r_tmp == r: + r_tmp = self.rng.sample(sorted(ent_in[answer]), 1)[0] + if (r_tmp != self.inverse_relations.get(r) if hasattr(self, 'inverse_relations') else r_tmp // 2 != r // 2 or r_tmp == r): r = r_tmp found = True break if not found: return True query_structure[-1][i] = r - answer = random.sample(list(ent_in[answer][r]), 1)[0] + answer = self.rng.sample(sorted(ent_in[answer][r]), 1)[0] if query_structure[0] == 'e': query_structure[0] = answer else: @@ -156,40 +159,10 @@ def achieve_answer(self, query: List[Union[str, List]], Private method for achieve_answer logic. @TODO: Document the code """ - assert isinstance(query[-1], list) - all_relation_flag = True - for ele in query[-1]: - # @TODO: unclear - if not isinstance(ele, int) or (ele == -1): - all_relation_flag = False - break - if all_relation_flag: - if isinstance(query[0], int): - # @TODO: unclear - ent_set = set([query[0]]) - else: - ent_set = self.achieve_answer(query[0], ent_in, ent_out) - for i in range(len(query[-1])): - if query[-1][i] == -2: - ent_set = set(range(len(ent_in))) - ent_set - else: - ent_set_traverse = set() - for ent in ent_set: - ent_set_traverse = ent_set_traverse.union(ent_out[ent][query[-1][i]]) - ent_set = ent_set_traverse - else: - ent_set = self.achieve_answer(query[0], ent_in, ent_out) - union_flag = False - if len(query[-1]) == 1 and query[-1][0] == -1: - union_flag = True - for i in range(1, len(query)): - if not union_flag: - ent_set = ent_set.intersection(self.achieve_answer(query[i], ent_in, ent_out)) - else: - if i == len(query) - 1: - continue - ent_set = ent_set.union(self.achieve_answer(query[i], ent_in, ent_out)) - return ent_set + num_entities = getattr(self, 'num_entities', None) + if num_entities is None: + num_entities = len(self.ent2id) if self.ent2id is not None else max(set(ent_in) | set(ent_out), default=-1) + 1 + return exact_answers(compile_query(query), ent_out, num_entities) def write_links(self, ent_out, small_ent_out): # @TODO: Explain why this is needed diff --git a/dicee/read_preprocess_save_load_kg/read_from_disk.py b/dicee/read_preprocess_save_load_kg/read_from_disk.py index 1a9fee2bb..7168e2e4a 100644 --- a/dicee/read_preprocess_save_load_kg/read_from_disk.py +++ b/dicee/read_preprocess_save_load_kg/read_from_disk.py @@ -1,5 +1,6 @@ import glob import logging +from pathlib import Path import numpy as np import pandas as pd @@ -50,15 +51,18 @@ def start(self) -> None: self.kg.raw_test_set = None elif self.kg.dataset_dir: for i in glob.glob(self.kg.dataset_dir + '/*'): - if 'train' in i: + # Route by the file name only: a parent folder named like "pytest-..." or "train-..." must not + # decide the split. + name = Path(i).name + if 'train' in name: self.kg.raw_train_set = read_from_disk(i, self.kg.read_only_few, self.kg.sample_triples_ratio, backend=self.kg.backend, separator=self.kg.separator) if self.kg.add_noise_rate: self.add_noisy_triples_into_training() - elif 'test' in i and self.kg.eval_model is not None: + elif 'test' in name and self.kg.eval_model is not None: self.kg.raw_test_set = read_from_disk(i, backend=self.kg.backend, separator=self.kg.separator) - elif 'valid' in i and self.kg.eval_model is not None: + elif 'valid' in name and self.kg.eval_model is not None: self.kg.raw_valid_set = read_from_disk(i, backend=self.kg.backend, separator=self.kg.separator) else: logger.warning(f'Not processed data: {i}') diff --git a/dicee/scripts/evaluate_query_methods.py b/dicee/scripts/evaluate_query_methods.py new file mode 100644 index 000000000..077c9cf7a --- /dev/null +++ b/dicee/scripts/evaluate_query_methods.py @@ -0,0 +1,42 @@ +"""Evaluate complex-query answering methods from JSON recipes. + + python -m dicee.scripts.evaluate_query_methods --manifest recipe.json --output DIR + +A manifest holds one recipe or a list of recipes with unique ``id`` fields; +results go to ``DIR/ID``. The reproducible benchmark suites, with frozen inputs +and verification, run through ``python -m benchmarks.cqa``. +""" + +import argparse +import json +from pathlib import Path + +from dicee.query_answering.method_evaluation import evaluate_method + + +def main(argv: list[str] | None = None) -> None: + """Evaluate each recipe of ``--manifest`` into ``--output/ID``.""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--manifest', type=Path, required=True, help='JSON recipe or list of recipes') + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--device', default='cpu') + parser.add_argument('--split', choices=('valid', 'test'), default='test') + parser.add_argument('--max-queries-per-shape', type=int, help='Uniform sample of each query type') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--threads', type=int) + args = parser.parse_args(argv) + recipes = json.loads(args.manifest.read_text()) + recipes = recipes if isinstance(recipes, list) else [recipes] + names = [recipe.get('id', f'{recipe["method"]}-{recipe["dataset"]}') for recipe in recipes] + if len(set(names)) != len(names) or any(Path(name).name != name or name in ('.', '..') for name in names): + parser.error('Recipe IDs must be unique, plain directory names') + for name, recipe in zip(names, recipes): + recipe = dict(recipe) + query_order = recipe.pop('query_order', 'relation') + report = evaluate_method(recipe, output=args.output / name, device=args.device, split=args.split, + limit=args.max_queries_per_shape, seed=args.seed, threads=args.threads, query_order=query_order) + print(json.dumps(dict(id=name, queries=report['queries'], seconds=report['seconds'], averages=report['averages'])), flush=True) + + +if __name__ == '__main__': + main() diff --git a/dicee/scripts/run.py b/dicee/scripts/run.py index c37429e40..fd3f93a75 100755 --- a/dicee/scripts/run.py +++ b/dicee/scripts/run.py @@ -56,6 +56,18 @@ def get_default_arguments(description=None): parser.add_argument('--flock_compact_state', action=argparse.BooleanOptionalAction, default=True) parser.add_argument('--flock_compile_sampler', action=argparse.BooleanOptionalAction, default=True) parser.add_argument('--flock_pack_walks', action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--kgicl_checkpoint", default=None, help="Official KG-ICL model_best.tar path") + parser.add_argument("--kgicl_dim", type=int, default=32) + parser.add_argument("--kgicl_attn_dim", type=int, default=5) + parser.add_argument("--kgicl_num_layers", type=int, default=6) + parser.add_argument("--kgicl_prompt_layers", type=int, default=3) + parser.add_argument("--kgicl_prompt_hops", type=int, default=3) + parser.add_argument("--kgicl_shots", type=int, default=5) + parser.add_argument("--kgicl_prompt_open_nodes", type=int, default=50) + parser.add_argument("--kgicl_prompt_seed", type=int, default=0) + parser.add_argument("--kgicl_masked_distances", type=int, nargs="*", default=None, + help="Zero the scores of entities first reached at these hop distances (upstream's optional answer-distance mask)") + parser.add_argument("--kgicl_query_batch_size", type=int, default=8) parser.add_argument('--graph_inference_backend', choices=['auto', 'torch', 'triton'], default='auto') parser.add_argument('--graph_relation_cache_mb', type=int, default=64) parser.add_argument('--graph_projection_cache_mb', type=int, default=64) @@ -66,7 +78,7 @@ def get_default_arguments(description=None): # Model related arguments parser.add_argument("--model", type=str, default="Keci", - choices=["ULTRA", "TRIX", "TRIXRelation", "Flock", "FlockRelation", "ComplEx", "Keci", "KeciTransformer", "CKeci", "ConEx", "AConEx", "ConvQ", "AConvQ", "ConvO", "AConvO", "QMult", + choices=["ULTRA", "TRIX", "TRIXRelation", "Flock", "FlockRelation", "KGICL", "ComplEx", "Keci", "KeciTransformer", "CKeci", "ConEx", "AConEx", "ConvQ", "AConvQ", "ConvO", "AConvO", "QMult", "OMult", "Shallom", "DistMult", "TransE", "MuRE", "TransH", "RotatE", "DualE", "BytE", "CoKE", "Pykeen_MuRE", "Pykeen_QuatE", "Pykeen_DistMult", "Pykeen_BoxE", "Pykeen_CP", @@ -148,7 +160,7 @@ def get_default_arguments(description=None): choices=["None", "train", "train_val", "train_val_test", "test", "val_test", "val", "train_test"], help='Evaluating link prediction performance on data splits. ') parser.add_argument("--eval_tie_policy", choices=TIE_POLICIES, default="sort", - help="Prediction ties: legacy sort ordering, best rank (optimistic), " + help="Prediction ties: sort ordering, best rank (optimistic), " "uniform random tied rank, or worst rank (pessimistic).") parser.add_argument("--eval_tie_seed", type=int, default=None, help="Seed for independent random tie-breaking; defaults to random_seed.") diff --git a/dicee/static_funcs.py b/dicee/static_funcs.py index 635dfa3e5..20735098a 100644 --- a/dicee/static_funcs.py +++ b/dicee/static_funcs.py @@ -30,6 +30,7 @@ from .models.flock import Flock, FlockRelation from .models.fsdp_models import FSDPShardedEntityModel, create_fsdp_sharded_model_class from .models.graph_model import GraphKGE +from .models.kgicl import KGICL from .models.pykeen_models import PykeenKGE from .models.transformers import BytE from .models.trix import TRIX, TRIXRelation @@ -42,6 +43,7 @@ 'ULTRA': (ULTRA, 'EntityPrediction'), 'Flock': (Flock, 'EntityPrediction'), 'FlockRelation': (FlockRelation, 'RelationPrediction'), + 'KGICL': (KGICL, 'EntityPrediction'), 'TRIX': (TRIX, 'EntityPrediction'), 'TRIXRelation': (TRIXRelation, 'RelationPrediction'), 'Shallom': (Shallom, 'RelationPrediction'), diff --git a/dicee/static_preprocess_funcs.py b/dicee/static_preprocess_funcs.py index 13b499351..4e271a473 100644 --- a/dicee/static_preprocess_funcs.py +++ b/dicee/static_preprocess_funcs.py @@ -84,7 +84,7 @@ def preprocesses_input_args(args): temperature = getattr(args, "adversarial_temperature", None) if temperature is not None and (not np.isfinite(temperature) or temperature < 0): raise ValueError("adversarial_temperature must be finite and nonnegative") - if args.model in ("ULTRA", "TRIX", "TRIXRelation", "Flock", "FlockRelation"): + if args.model in ("ULTRA", "TRIX", "TRIXRelation", "Flock", "FlockRelation", "KGICL"): if args.trainer not in ("torchCPUTrainer", "PL"): raise ValueError(f"{args.model} currently supports CPU/single GPU native and Lightning trainers") if args.byte_pair_encoding or args.num_folds_for_cv or args.save_embeddings_as_csv: diff --git a/dicee/trainer/dice_trainer.py b/dicee/trainer/dice_trainer.py index fc7b0cbba..3842981ee 100644 --- a/dicee/trainer/dice_trainer.py +++ b/dicee/trainer/dice_trainer.py @@ -125,7 +125,7 @@ def initialize_trainer( # Fall back to CPU when CUDA is unavailable or its context is broken. # _disable_cuda_in_process() was already called above when needed. _default_accelerator = "cpu" if not torch.cuda.is_available() else "auto" - trainer = pl.Trainer(devices=kwargs.get("devices", 1 if args.model in ("ULTRA", "TRIX", "TRIXRelation", "Flock", "FlockRelation") else "auto"), accelerator=kwargs.get("accelerator", _default_accelerator), + trainer = pl.Trainer(devices=kwargs.get("devices", 1 if args.model in ("ULTRA", "TRIX", "TRIXRelation", "Flock", "FlockRelation", "KGICL") else "auto"), accelerator=kwargs.get("accelerator", _default_accelerator), strategy=kwargs.get("strategy", "auto"), num_nodes=kwargs.get("num_nodes", 1), precision=kwargs.get("precision", None), diff --git a/docs/guides/multi_hop_queries.md b/docs/guides/multi_hop_queries.md index 60bb5f856..69d0ed2e3 100644 --- a/docs/guides/multi_hop_queries.md +++ b/docs/guides/multi_hop_queries.md @@ -1,6 +1,10 @@ -# Multi-Hop Query Answering +# Complex Query Answering (CQA) -This guide explains how to use DICE Embeddings for complex multi-hop query answering using EPFO (Existential Positive First-Order) queries. +This guide explains how to use DICE Embeddings for complex multi-hop query answering using EPFO (Existential Positive First-Order) queries and their supported negated forms. + +Ordinary KGEs and ULTRA/TRIX/Flock share the same evaluator. Scores default to +sigmoid memberships (`use_logits=False`), and every query respects the answer +limit `k`. Use `beam_size` to control intermediate search independently. ## Table of Contents @@ -9,12 +13,13 @@ This guide explains how to use DICE Embeddings for complex multi-hop query answe - [Usage Examples](#usage-examples) - [Advanced Patterns](#advanced-patterns) - [Performance Tips](#performance-tips) +- [Training and evaluating query methods](#training-and-evaluating-query-methods) --- ## Overview -Multi-hop query answering allows you to ask complex questions that require reasoning over multiple relationships in a knowledge graph. DICE Embeddings supports **9 query types** covering projections, intersections, and unions. +Multi-hop query answering combines relationships in a knowledge graph. DICE supports the **14 UltraQuery query types**, including `2in`, `3in`, `inp`, `pin`, and `pni` for negation, and the additional `4p` and `4i` types used by +H. ### Supported Query Types @@ -396,15 +401,15 @@ for query_id, preds in results.items(): The `tnorm` parameter controls how conjunctions are computed: ```python -# Minimum t-norm (default, most commonly used) +# Minimum t-norm model.answer_multi_hop_query(..., tnorm="min") -# Product t-norm (multiplicative semantics) +# Product t-norm (default, multiplicative semantics) model.answer_multi_hop_query(..., tnorm="prod") ``` **When to use:** -- **`min`**: Standard choice, preserves highest score in conjunction +- **`min`**: Takes the lower membership in a conjunction - **`prod`**: More strict, all conditions must have high scores ### Filtering Results @@ -580,7 +585,7 @@ candidate_drugs = model.answer_multi_hop_query( ### Limitations -1. **Negation not supported** — Cannot express "NOT" queries +1. **Fixed query structures** — Only the 16 types above; negation is available through `2in`, `3in`, `inp`, `pin`, and `pni` 2. **Numerical constraints not supported** — Cannot express "> 5" or "between X and Y" 3. **Recursive queries not supported** — Cannot express "transitive closure" 4. **Performance degrades with query complexity** — 3-hop queries slower than 1-hop @@ -621,7 +626,7 @@ candidate_drugs = model.answer_multi_hop_query( **Problem:** Queries take too long **Solutions:** -1. Reduce `k` parameter +1. Reduce `beam_size` to limit intermediate candidates; `k` controls only the returned answers 2. Use simpler query types when possible 3. Ensure model is on GPU: `model.to("cuda")` 4. Cache repeated query patterns @@ -632,7 +637,7 @@ candidate_drugs = model.answer_multi_hop_query( - **Tests:** [test_answer_multi_hop_query.py](../../tests/test_answer_multi_hop_query.py) — Complete examples - **Research:** [Query2Box paper](https://arxiv.org/abs/2002.05969) — Theoretical foundation -- **Benchmarks:** [examples/multi_hop_query_answering/](../../examples/multi_hop_query_answering/) — Performance evaluation +- **Benchmarks:** [benchmarks/cqa](../../benchmarks/cqa/README.md) — Reproducible UltraQuery and +H evaluation --- @@ -646,12 +651,50 @@ candidate_drugs = model.answer_multi_hop_query( | Follow chain of relations | `2p`, `3p` | | Find entities satisfying AND conditions | `2i`, `3i` | | Find entities satisfying OR conditions | `2u` | -| Complex patterns | `ip`, `pi`, `up` | +| Exclude entities (NOT) | `2in`, `3in`, `inp`, `pin`, `pni` | +| Complex patterns | `ip`, `pi`, `up`, `4p`, `4i` | **Best Practices:** - ✅ Start with simple queries (1p, 2p) -- ✅ Use `tnorm="min"` as default +- ✅ Compare `tnorm="prod"` and `tnorm="min"` on validation queries - ✅ Train with `AllvsAll` for best reasoning - ✅ Use high `embedding_dim` (≥ 256) - ✅ Validate query syntax carefully - ✅ Benchmark on known ground truth first + +## Training and evaluating query methods + +Fit adapters with `python -m dicee.query_answering fit --help`; prepare source +queries with `python -m dicee.query_answering prepare --help`. The backbone stays +frozen, and the saved adapter records its configuration and backbone fingerprint. +The default is linear `context_scores` (16 parameters) for every backbone; +`--feature-mode context` or `global` selects simpler alternatives. + +The `dicee.query_answering.methods` package exposes ConE, CLMPT, CQD/CQD-Hybrid, +GNN-QE, QTO, UltraQuery, and `load_method()` for their released checkpoints. +`dicee.query_answering.evaluate_method()` evaluates these methods and our +ULTRA/TRIX adapters through the same filtered ranking and checkpoint machinery. +It accepts a recipe with `method`, `checkpoint`, `dataset`, and `root`; KGFM recipes +also specify `adapters`, per-type `operators`, and `selection_protocol`. + +The CLI evaluates a JSON recipe, or a list of recipes with unique `id` fields: + +```bash +python -m dicee.scripts.evaluate_query_methods \ + --manifest recipe.json --output results/cqa-pilot \ + --split valid --max-queries-per-shape 2 --device cuda +``` + +It supports every loaded dataset family, including UltraQuery and +H. The +[benchmark harness](../../benchmarks/cqa/README.md) selects methods, datasets and +query types from the public recipes, freezes query batches, verifies upstream +parity: + +```bash +python -m benchmarks.cqa plus_h evaluate --methods ultra-adapter --datasets FB15k237+H \ + --query-types 2p 3p --output results/cqa-pilot --split valid --max-queries-per-shape 2 +``` + +Query groups are `all`, `epfo`, and `negation`. Shipped +H CQD/CQD-Hybrid recipes +default to all 16 types with atomic negation enabled. Adapter definitions are in +`score_adapter.py`; fitting and source-query preparation are in `adapter_training.py`. diff --git a/docs/kgfm_benchmarks.md b/docs/kgfm_benchmarks.md index 687226279..5e4d5f65c 100644 --- a/docs/kgfm_benchmarks.md +++ b/docs/kgfm_benchmarks.md @@ -23,6 +23,10 @@ lists all 50 graphs. It includes FB15k-237, WN18RR, YAGO3-10, and NELL995; KINSHIP, UMLS, and Countries are absent. **No** indicates transfer outside a checkpoint's documented training mixture. The original 4g/50g run records retain the earlier metadata; these corrected labels do not change the measured scores. +KG-ICL was pretrained on FB V1, NELL V1, and CoDEx-small +([configuration](https://github.com/nju-websoft/KG-ICL/blob/6a3166e347ae468acdfb30a70a2cf3608b66b8f1/src/pretrain.py#L72)). +FB V1 and NELL V1 are GraIL subgraphs of FB15k-237 and NELL-995, so **Partial** +marks those targets for KG-ICL (counts below). ## Checkpoints and evaluation settings @@ -33,6 +37,7 @@ the earlier metadata; these corrected labels do not change the measured scores. | ULTRA-50g | [ultra_50g.pth](https://github.com/DeepGraphLearning/ULTRA/blob/427966ad8ed60420eef034063d44f3153addff90/ckpts/ultra_50g.pth) | `checkpoints/ultra_50g.pth` | [ULTRA](ultra.md) | | TRIX | [entity_prediction.pth](https://github.com/yuchengz99/TRIX/blob/7596e14eefefe89e61396205a0550172cadeddb0/entity_prediction.pth) | `checkpoints/trix/entity_prediction.pth` | [TRIX](trix.md) | | Flock | [flock_entity.pth](https://github.com/jw9730/flock/blob/f35103d25a78bdf4075de5c673a51de4979aa4d7/checkpoints/flock_entity.pth) | `checkpoints/flock/flock_entity.pth` | [Flock](flock.md) | +| KG-ICL | [KG-ICL-6L/model_best.tar](https://github.com/nju-websoft/KG-ICL/blob/6a3166e347ae468acdfb30a70a2cf3608b66b8f1/checkpoint/KG-ICL-6L/model_best.tar) | `checkpoints/kgicl/KG-ICL-6L/model_best.tar` | [KG-ICL](kgicl.md) | Runs use float32, TF32 disabled on GPU, four CPU threads, 128 test triples per batch, and seed 42. ULTRA/TRIX on UMLS use batches of 8 and seed 1. @@ -40,18 +45,20 @@ Flock uses 128 base walks of length 128, six refinements, and one prediction per query: a single-seed result, not an ensemble. Its query order and batch boundaries affect sampling; see the [sampling notes](flock.md#checkpoints-and-zero-shot-evaluation). -| Dataset | ULTRA device / query batch | TRIX device / query batch | Flock device / query batch | -|---|---|---|---| -| Countries-S1 / S2 / S3, UMLS | CPU / 8 | CPU / 8 | CUDA / 1 | -| WN18RR | CUDA / 8 | CUDA / 4 | CUDA / 1 | -| YAGO3-10 | CUDA / 2 | CUDA / 2 | CUDA / 1 | -| FB15k-237 | CUDA / 4 | CUDA / 2 | CUDA / 1 | -| KINSHIP | CUDA / 8 | CUDA / 2 | CUDA / 1 | -| NELL-995-h25 / h50 / h75 / h100 | CUDA / 4 | CUDA / 2 | CUDA / 1 | - +| Dataset | ULTRA device / query batch | TRIX device / query batch | Flock device / query batch | KG-ICL device / query batch | +|---|---|---|---|---| +| Countries-S1 / S2 / S3, UMLS | CPU / 8 | CPU / 8 | CUDA / 1 | CUDA / 16 | +| WN18RR | CUDA / 8 | CUDA / 4 | CUDA / 1 | CUDA / 16 | +| YAGO3-10 | CUDA / 2 | CUDA / 2 | CUDA / 1 | CUDA / 16 | +| FB15k-237 | CUDA / 4 | CUDA / 2 | CUDA / 1 | CUDA / 16 | +| KINSHIP | CUDA / 8 | CUDA / 2 | CUDA / 1 | CUDA / 16 | +| NELL-995-h25 / h50 / h75 / h100 | CUDA / 4 | CUDA / 2 | CUDA / 1 | CUDA / 16 | + +KG-ICL uses prompt seed 0 and no answer-distance mask; its official evaluation +protocol differs, see the [KG-ICL guide](kgicl.md#scores-and-the-official-evaluation-protocol). The September 10 runs used PyTorch 2.5.1, Triton 3.1.0, an AMD Ryzen 7 7800X3D CPU, and an RTX 4070 Ti SUPER GPU. KINSHIP, all four NELL variants, and the ULTRA-4g/50g runs (September 21) use -PyTorch 2.9.1+cu128, Triton 3.5.1, and an RTX 5070 Laptop GPU. These are accuracy +PyTorch 2.9.1+cu128, Triton 3.5.1, and an RTX 5070 Laptop GPU, as do the KG-ICL runs (October 5). These are accuracy results; controlled timings are in the [inference speed comparison](kgfm_inference.md). ## KINSHIP and NELL variants @@ -94,6 +101,22 @@ zero-shot held-out estimates. The upstream data commit predates both checkpoints file hashes and all split comparisons are saved in `Experiments/nell-pretraining-overlap/audit.json`. +**Pretraining overlap (KG-ICL):** exact triple matches against the training and +validation files of FB V1, NELL V1, and CoDEx-small in the official `datasets.zip` +at the pinned commit. GraIL's NELL V1 names entities `concept:type:name`; they are +compared as `concept_type_name`, the NELL-995 variants' spelling. + +| Target | Pretraining graph | Distinct test facts present | Percentage | Shared entities | +|---|---|---:|---:|---:| +| FB15k-237 | FB V1 | 0 / 20,466 | 0.00% | 1,594 | +| NELL-995-h25 | NELL V1 | 257 / 9,185 | 2.80% | 2,929 | +| NELL-995-h50 | NELL V1 | 200 / 5,392 | 3.71% | 2,825 | +| NELL-995-h75 | NELL V1 | 192 / 4,389 | 4.37% | 2,750 | +| NELL-995-h100 | NELL V1 | 174 / 3,746 | 4.64% | 2,488 | + +All 4,734 FB V1 training and validation facts are FB15k-237 training facts. No +other README target shares entities with KG-ICL's pretraining graphs. + ## Reproduction Place the checkpoints at the paths above and datasets under `KGs//` @@ -129,7 +152,18 @@ python benchmarks/kgfm_zero_shot.py --model ULTRA --ultra-checkpoint 4g \ --output Experiments/kgfm-ultra-checkpoints-20260921/NELL-995-h50/ULTRA-4g ``` -Use a separate output per checkpoint. For other datasets, use the settings above. The +Use a separate output per checkpoint. For other datasets, use the settings above. +KG-ICL runs every dataset with the same settings; `--prompt-seed` selects the +prompt sampling seed (default 0) and `--distance-mask` applies the official +answer-distance heuristic, which the README results do not use: + +```bash +python benchmarks/kgfm_zero_shot.py --model KGICL --dataset FB15k-237 --device cuda:0 \ + --query-batch-size 16 --batch-size 128 --threads 4 --seed 42 --tie-policy pessimistic \ + --output Experiments/kgfm-kgicl-20261005/FB15k-237/KGICL +``` + +The [runner](../benchmarks/kgfm_zero_shot.py) verifies unchanged weights and saves progress after each batch. An identical invocation resumes; changed settings or source require a new output directory. Local reports include metrics, exact diff --git a/docs/kgfm_inference.md b/docs/kgfm_inference.md index 50ba24e9e..436edb842 100644 --- a/docs/kgfm_inference.md +++ b/docs/kgfm_inference.md @@ -82,6 +82,72 @@ YAGO3-10 Flock had higher timing variance: official native repeats ranged 394–583 ms/query (DICE 65–67), and replay repeats ranged 76–170 ms/query (DICE 47–54). Treat its reported median speedups as approximate. +## KG-ICL + +Measured on 2026-10-05 against the [pinned official KG-ICL](https://github.com/nju-websoft/KG-ICL/tree/6a3166e347ae468acdfb30a70a2cf3608b66b8f1) +with the [bug-fix patch](../tests/fixtures/kgicl/upstream-fixes.patch) applied (see the [KG-ICL guide](kgicl.md)), +on different hardware than the table above: an RTX 5070 Laptop GPU (8 GiB) and +an Intel Core i7-14700HX. + +| Dataset | Query batch | Official ms/query | DICE ms/query | Speedup | CUDA MiB | +|---|---:|---:|---:|---:|---:| +| FB15k-237 | 8 | 68.25 | 1.37 | 49.76× | 3096 → 177 | +| WN18RR | 16 | 3.63 | 0.82 | 4.43× | 924 → 326 | +| YAGO3-10† | 2 | 277.71 | 7.80 | 35.58× | 3382 → 373 | + +† Validated against the official model in float64, see below. + +The workload matches the table above: 128 test triples (seed 42), head and +tail prediction, and the complete training graph plus inverse edges. Both +implementations read the same prompt graphs, DICE's seeded samples written in +the official case format, and apply the official answer-distance mask. The +official code runs in its own environment (Python 3.9, PyTorch 2.7.1 with CUDA +12.8, torch-scatter 2.1.2); DICE uses PyTorch 2.9.1 with CUDA 12.8 and Triton +3.5.1. Both use float32 with TF32 disabled and four CPU threads, with two +warmups and five timed repetitions (medians), one process at a time. The +driver's GPU process listing was unavailable on this machine +(`--allow-unchecked-gpu`); no other compute job ran. Query batches are the +largest the official code fits in 8 GiB; it ran out of GPU memory on YAGO3-10 +with four queries. + +Prompt graph extraction is preprocessing for both implementations. The +official code assembles and encodes the prompt graphs of every batch, while +DICE encodes each relation's prompts once and caches them, as ULTRA caches +relation representations. Without that cache, DICE takes 8.97 ms/query on +FB15k-237. DICE gains from larger batches: with 16 queries per pass it takes +1.19 ms/query on FB15k-237 and 4.82 ms/query on YAGO3-10, with scores +bitwise identical to the table's runs. + +Scores agree within `atol=2e-4, rtol=2e-4` on FB15k-237 (maximum absolute error +`1.84e-4`) and WN18RR (`7.25e-5`), and filtered MRR and hit rates match (MRR +within `5e-8`). On YAGO3-10, float32 scores differ by up to `2.7e-3`: one entity +has 61,044 incoming edges, and long float32 sums depend on their order. +Against the official model in float64, run with one query per batch because two +do not fit in float64, DICE's maximum error is `2.65e-5` (mean `6.9e-7`) and +the official float32 code's is `2.73e-3` (mean `9.7e-5`). DICE's MRR equals the +float64 value; hit rates match. + +### Query answering rows + +Warm atomic rows in the query evaluator under deterministic algorithms, as in +complex query answering: batches of 8 random head–relation pairs scored against +all entities, on the same laptop GPU, with the KG-ICL recipes' 512 MiB cache of +per-layer relation tables (`projection_cache_mb`): + +| Inference graph | Entities | Relations with inverses | Edges | ms/row | First pass over all relations | +|---|---:|---:|---:|---:|---:| +| FB15k237+H | 14,505 | 474 | 544,230 | 1.01 | 16.9 s | +| NELL995+H | 63,361 | 400 | 228,426 | 1.71 | 8.4 s | +| ICEWS18+H | 20,840 | 500 | 426,608 | 1.04 | 20.0 s | +| UltraQuery FB15k237 | 14,505 | 474 | 544,230 | 0.98 | 14.1 s | +| UltraQuery NELL995 | 63,361 | 400 | 228,426 | 1.68 | 7.9 s | +| UltraQuery inductive FB15k237 (550) | 13,438 | 312 | 136,560 | 0.54 | 5.7 s | +| UltraQuery WikiTopics art | 10,000 | 129 | 54,524 | 0.55 | 2.7 s | + +The first pass samples and encodes the prompt graphs of every relation. With the +default 64 MiB table cache, graphs with hundreds of relations recompute tables: +FB15k237+H then takes 2.55 ms/row. + ## Reproduce Use the pinned official checkout with PyTorch 2.5.1/CUDA 12.4, PyG 2.4.0, @@ -101,3 +167,18 @@ python benchmarks/kgfm_upstream.py --model ULTRA \ Use the table’s batch sizes for the other pairs. For Flock, use `--queries 32 --query-batch-size 1 --walk-num 128 --replay-queries 8`. All generated JSON, score tensors, and walk records land in the `Experiments/` directories. + +KG-ICL's official worker runs in its own environment, passed with +`--upstream-python`: Python 3.9 with PyTorch, a matching torch-scatter build, +NumPy, SciPy and NetworkX. The runner applies the bug-fix patch to a temporary +copy of the pinned checkout; `--upstream-variant unpatched` times the released +code. Where `nvidia-smi` cannot list GPU processes, `--allow-unchecked-gpu` +records that instead of aborting; make sure no other GPU work runs. + +```bash +python benchmarks/kgfm_upstream.py --model KGICL \ + --upstream-root /path/to/KG-ICL --upstream-python /path/to/kgicl-env/bin/python \ + --indexed-data Experiments/kgfm-kgicl-20261005/FB15k-237/KGICL \ + --queries 128 --query-batch-size 8 --repeats 5 --warmups 2 \ + --output Experiments/official-comparison/FB15k-237/KGICL +``` diff --git a/docs/kgicl.md b/docs/kgicl.md new file mode 100644 index 000000000..6bf5ac069 --- /dev/null +++ b/docs/kgicl.md @@ -0,0 +1,366 @@ +# KG-ICL in DICE + +[Paper](https://arxiv.org/abs/2410.12288) · +[Pinned official implementation](https://github.com/nju-websoft/KG-ICL/tree/6a3166e347ae468acdfb30a70a2cf3608b66b8f1) · +[Bug-fix patch](../tests/fixtures/kgicl/upstream-fixes.patch) + +KG-ICL answers a query `(s, q, ?)` in context. It samples a few example facts +`(u, q, v)` of the query relation, extracts a prompt graph around each example +and encodes the relations of these prompt graphs. The mean prompt encoding +initializes the relations of a query-conditioned reasoner that expands one hop +per layer from `s` over the inference graph. DICE provides `KGICL` for entity +prediction with the graph, training and reload interfaces of [ULTRA](ultra.md), +[TRIX](trix.md) and [Flock](flock.md). The official `KG-ICL-4L`, `KG-ICL-5L` and +`KG-ICL-6L` checkpoints load without renaming or dropping parameters. Only +PyTorch is required; Triton adds the fused CUDA inference path. NetworkX, +torch-scatter and the official code are needed only to regenerate fixtures. + +## Checkpoints and zero-shot evaluation + +```bash +mkdir -p checkpoints/kgicl/KG-ICL-6L +wget -P checkpoints/kgicl/KG-ICL-6L https://raw.githubusercontent.com/nju-websoft/KG-ICL/6a3166e347ae468acdfb30a70a2cf3608b66b8f1/checkpoint/KG-ICL-6L/model_best.tar +sha256sum checkpoints/kgicl/KG-ICL-6L/model_best.tar +# 80f885ffa2c0175cd6ed829af99ae45177fdb5f77302c7648591bbdf62bc1c81 + +python -m dicee --model KGICL --dataset_dir KGs/UMLS \ + --kgicl_checkpoint checkpoints/kgicl/KG-ICL-6L/model_best.tar --num_epochs 0 \ + --trainer torchCPUTrainer --scoring_technique NegSample \ + --batch_size 8 --eval_model test \ + --path_to_store_single_run Experiments/kgicl-zero-shot +``` + +The directory contains `train.txt`, `test.txt` and optionally `valid.txt` in +`(head, relation, tail)` order. Reasoning and prompt graphs use training facts +only. Validation and test facts are used for filtered evaluation. The released +`model_best.tar` holds `{"state_dict", "optimizer_state_dict", "epoch_id"}`; +`load_pretrained()` also accepts a `{"model": ...}` wrapper or a bare state +dictionary. Keys and shapes are checked before loading. Fine-tuning starts with +a new optimizer. + +| Setting | Default | Meaning | +|---|---:|---| +| `kgicl_dim` | 32 | Hidden dimension | +| `kgicl_attn_dim` | 5 | Reasoner attention dimension | +| `kgicl_num_layers` | 6 | Reasoner layers: 4, 5 or 6 for the official checkpoints | +| `kgicl_prompt_layers` | 3 | Prompt encoder layers | +| `kgicl_prompt_hops` | 3 | Prompt distance bound `k`; sets the token table size | +| `kgicl_shots` | 5 | Prompt graphs per query relation | +| `kgicl_prompt_open_nodes` | 50 | Further entities per prompt graph, see below | +| `kgicl_prompt_seed` | 0 | Seed of example and entity sampling | +| `kgicl_masked_distances` | unset | Hop distances whose entities score zero | +| `kgicl_query_batch_size` | 8 | Queries per batched GPU reasoning pass | + +The first five settings determine checkpoint shapes; leave them at their +defaults for `KG-ICL-6L`. Generic `embedding_dim` does not control KG-ICL. +`graph_relation_cache_mb` bounds the cached prompt encodings and +`graph_projection_cache_mb` the per-layer relation tables of the fused path. + +## Prompt graphs + +DICE builds prompt graphs from the attached graph as the official data +processing builds them (`utils_all.py`, `enclosing=False`): + +- Examples are up to `kgicl_shots` distinct facts `(u, q, v)` of the direct + relation `q`. With `n < shots` facts, shot `j` uses example `j mod n`. +- Distances are undirected hop counts in the attached graph, computed up to + `k = 3` from `u` and from `v`. The prompt graph contains `u`, `v`, every + entity `x` within `k` of both with `dist(x,u) + dist(x,v) <= k`, and up to 50 + further entities within `k` of both. It contains every fact among them. +- Entity tokens are `(dist(x,u), dist(x,v))`. `u` receives the head token and + `v` the tail token. An inverse query relation `q⁻` reuses the examples of `q` + with head and tail tokens swapped; the other tokens keep their `(u, v)` order, + as in the released code. +- A relation without facts uses the official fallback: two nodes joined by one + query-relation edge. + +The official code samples examples and further entities with unseeded NumPy +calls once, offline. DICE draws them from CPU generators seeded with +`sha256("kgicl-prompt:{seed}:{relation}")`, for the prompt seed and the public +ID of the direct relation; further entities additionally use the example index. +Examples are drawn from the relation's facts sorted by `(head, tail)`. A graph, +relation and seed therefore always produce the same prompt graphs, independently +of query batches, call order and other relations. Prompt graphs are built on +first use. Their encodings are cached per relation during inference and cleared +when the graph, weights, device or settings change. During training, each shot +draws one of the relation's examples at random, as the official training does. + +## Scores and the official evaluation protocol + +Entities the reasoner does not reach within `kgicl_num_layers` hops score exactly +zero, as in the official code. The official evaluation differs from DICE's +benchmark protocol in three ways, which DICE exposes but does not apply by default: + +- **Inference graph.** Official test queries reason over training plus + validation facts; prompt graphs use training facts only. DICE uses the graph + passed to `set_graph()` for both. +- **Distance mask.** Official evaluation computes, from validation facts, the + fraction of answers at each hop distance and zeroes entities first reached at + distances below 2% (always including the query head). Use + `--kgicl_masked_distances 0 5 6` or `answer_distance_rates()` with + `benchmarks/kgfm_zero_shot.py --distance-mask` to apply it. +- **Ties.** Official ranks break exact ties by entity ID (`rankdata(method='ordinal')`). + +## Training and fine-tuning + +Entity prediction supports `NegSample`, `FixedNegSample`, `KvsAll`, `1vsAll`, +`1vsSample` and `KvsSample`, with grouped strict negatives and adversarial BCE: + +```bash +python -m dicee --model KGICL --dataset_dir KGs/UMLS \ + --kgicl_checkpoint checkpoints/kgicl/KG-ICL-6L/model_best.tar \ + --trainer torchCPUTrainer --scoring_technique NegSample \ + --strict_negative_sampling --adversarial_temperature 1 --neg_ratio 64 \ + --batch_size 8 --num_epochs 5 --optim Adam --lr 0.0005 +``` + +Training removes supervised facts and their inverses from the reasoning graph; +pair queries hide all their known answers. Prompt graphs keep all training facts, +as in the official training. Training mode samples RReLU slopes, applies the +prompt encoder's dropout and draws prompt examples at random. DICE trains with +its own objectives; it does not reproduce the official multi-graph pretraining +loop or its relation-dropout augmentation. Native CPU/single GPU and +single-device Lightning are supported. BPE, distributed training, +cross-validation and static embedding export are rejected. + +## Python use and experiment reloads + +```python +import torch +from dicee.models import KGICL + +facts = torch.tensor([[0, 0, 1], [1, 1, 2], [2, 0, 0], [0, 1, 2]]) +model = KGICL(dict(num_entities=3, num_relations=2)).load_pretrained( + "checkpoints/kgicl/KG-ICL-6L/model_best.tar") +model.set_graph(facts).eval() +with torch.no_grad(): + triples = model(torch.tensor([[0, 1, 2]])) + tails = model(torch.tensor([[0, 1]])) # (head, relation) + heads = model.forward_k_vs_all_heads(torch.tensor([[1, 2]])) # (relation, tail) +``` + +Scores are logits. Call `.eval()` for inference. Triples use DICE's +`(head, relation, tail)` order. Head prediction conditions on the converted +inverse relation, as the official evaluation does. `set_graph()` adds inverse +edges, deduplicates facts and resets prompt graphs; pass +`inverse_relations={direct_id: inverse_id}` for vocabularies with explicit +inverses. `prompt_graphs(relation)` returns the prompt graphs of an internal +relation ID (inverses are `r + R`), `encode_prompts(relation, graphs)` encodes +any prompt graphs, and `use_prompts({relation: graphs})` replays fixed prompt +graphs, for example ones recorded from the official loader. + +```python +from dicee import KGE + +kge = KGE(path="Experiments/kgicl-zero-shot") +tails = kge.predict_missing_tail_entity("entity_a", "relation_r") +``` + +Keep `kgicl_graph.pt` beside the weights, configuration and vocabularies when +moving an experiment. Prompt graphs are rebuilt from it with the saved seed. + +## Determinism and batching + +Inference is deterministic, and a row depends only on the graph, weights, +settings, head and relation: + +- The PyTorch path scores each query alone during inference, so its rows never + depend on the batch. On CUDA it is bitwise reproducible under + `torch.use_deterministic_algorithms(True)` (set `CUBLAS_WORKSPACE_CONFIG`). +- The fused CUDA path (Triton) keeps a dense entity-major `[entities, queries, dim]` + state with an active mask: messages leave only active entities, which + reproduces the official hop-by-hop expansion. One program aggregates the + incoming edges of one entity for a block of four queries, whose source states + are contiguous; the attention's source term `Ws_attn h` is projected once per + entity and layer instead of once per edge. Each row's aggregation and GRU + update use a fixed operation order without atomics, so rows are bitwise + identical for every query batch size and order, with or without deterministic + algorithms. Entities with more than 256 incoming edges are summed in 64-edge + partial sums that are added in order. Dense products use 3xTF32 tensor-core + dots, which keep float32 accuracy, on GPUs with TF32 support, and IEEE FMA + dots otherwise. +- Prompt encodings and relation tables are computed for one relation at a time. + +`kgicl_query_batch_size` only trades memory for speed on the fused path: each +query holds about `420 * entities` bytes of state, and 16 queries per pass are +about 13% faster than 8 on FB15k-237. See [KGFM inference speed](kgfm_inference.md#kg-icl) +for timings against the official code. + +## Upstream defects corrected in DICE + +DICE implements the corrected behavior. `tests/fixtures/kgicl/upstream-fixes.patch` +applies exactly these fixes to the pinned official code; all parity checks below +compare against the patched official code. Effects are measured below. + +| ID | Location | Defect | Correction | +|---|---|---|---| +| B1 | `EntityEncoder.py` | The reasoner's attention activation is a fresh `nn.RReLU()` per call. A new module is in training mode, so evaluation samples random negative slopes: predictions change between runs and exact ties are broken at random. | Use the model's mode: random slopes in training, the mean slope `(1/8 + 1/3) / 2` in evaluation, as RReLU defines. | +| B2 | `data_loader.py` | Evaluation shot `i` of the batch-wide relation list selects case `i mod n_cases`. For a relation with 2 to 4 cases, the selected cases depend on the relation's rank among the batch's relations, so prompts depend on the (shuffled) query batch. | Shot `j` of a relation selects case `j mod n_cases`. | +| B3 | `utils_all.py` | Distance labels are aligned with the node list but are paired with nodes in Python set order, and the loader reads label rows as local node IDs. Most prompt entities receive another entity's distance tokens. The pretraining data was processed with the same code. | Pair each entity with its own label; write rows in local-ID order. | +| B4 | `utils_all.py` | A self-loop pair `(a, a)` is looked up in both directions, so every self-loop fact appears twice in the prompt graph (doubling its degree). | Each fact appears once. | +| B5 | `data_loader.py` | For a self-loop example `(u, q, u)` the tail token goes to local node 1, an unrelated entity. | Head and tail tokens go to the example entity (the tail token overwrites the head token, matching the token assignment order). | + +No defect leaks test facts: prompt graphs come from training facts, and the +reasoning graph of test queries holds training and validation facts. + +### Measured effects + +Filtered MRR of the official code under the official protocol, relative to the +code with all five fixes, on the five verification datasets below. Each row +restores one defect: B1, B2 and B5 in the code, B3 and B4 in the prompt +processing. Rows with random or order-dependent results give the mean and +standard deviation over three query orders. + +| | WN V1 | FB V2 | NELL V2 | NL-0 | FB-25 | +|---|---:|---:|---:|---:|---:| +| MRR with all fixes | 0.7346 | 0.5635 | 0.6391 | 0.5655 | 0.4213 | +| Range over three query orders | 0.7345–0.7346 | 0.5634–0.5635 | 0.6390–0.6391 | 0.5654–0.5674 | 0.4213–0.4213 | +| B1 | −0.0008 ± 0.0006 | −0.0006 ± 0.0009 | −0.0031 ± 0.0022 | −0.0049 ± 0.0041 | +0.0001 ± 0.0002 | +| B1, pessimistic ties | −0.0008 ± 0.0006 | +0.0004 ± 0.0009 | −0.0001 ± 0.0022 | +0.0006 ± 0.0041 | +0.0002 ± 0.0002 | +| B2 | +0.0010 ± 0.0000 | +0.0003 ± 0.0003 | 0.0000 ± 0.0000 | −0.0002 ± 0.0000 | +0.0001 ± 0.0000 | +| B3 | −0.0014 | +0.0021 | −0.0029 | −0.0100 | +0.0052 | +| B4 | not present | not present | not present | −0.0010 | −0.0087 | +| B5 | not present | not present | not present | 0.0000 | +0.0012 | +| All five (released code) | +0.0027 ± 0.0011 | +0.0023 ± 0.0005 | −0.0039 ± 0.0021 | −0.0051 ± 0.0029 | −0.0022 ± 0.0002 | + +How many prompts each processing or loading defect touches, among the five +examples per relation that evaluation loads: + +| | WN V1 | FB V2 | NELL V2 | NL-0 | FB-25 | +|---|---:|---:|---:|---:|---:| +| Relations with 2 to 4 examples (B2) | 2 of 8 | 32 of 172 | 9 of 79 | 21 of 112 | 23 of 216 | +| Prompt entities with another entity's tokens (B3) | 237 of 396 | 25,236 of 43,692 | 14,745 of 27,634 | 17,090 of 33,480 | 52,901 of 98,725 | +| Prompt graphs with duplicated self-loop facts (B4) | 0 of 36 | 0 of 710 | 0 of 353 | 95 of 470 | 949 of 979 | +| Self-loop examples (B5) | 0 | 0 | 0 | 3 | 21 | + +- **B1** makes evaluation nondeterministic: the standard deviation of MRR over + three runs reaches 0.004 (NL-0). Its random slopes also break exact score ties + at random; under the official ranking, which orders ties by entity ID, this + lowers MRR on NL-0 and NELL V2, while pessimistic MRR changes by less than 0.001. +- **B2** changes MRR by at most 0.0010 here, but makes a query's prediction + depend on which relations share its batch. +- **B3** is the largest defect. The checkpoints were pretrained with the same + processing code (`datasets/utils_all.py` contains both B3 and B4), so the + correction changes prompts relative to training, and its effect varies in sign: + from −0.0100 to +0.0052 MRR. +- **B4** lowers MRR on graphs with self-loop facts in prompts (−0.0087 on FB-25, + where almost every prompt graph has them). +- **B5** affects only self-loop examples (+0.0012 on FB-25). + +Official-code differences of about 1e-4 are within the run-to-run noise of the +official CUDA scatter kernels (range row). DICE always implements the +corrected behavior; to evaluate the released behavior, run the official code. + +## Upstream behavior DICE keeps + +These choices are consistent between the official training and evaluation, and +the checkpoints depend on them; DICE matches them: + +- The prompt encoder multiplies entity messages by the symmetric degree + normalization twice, and the reasoner applies each relation layer norm twice. +- Unused parameters (`layer_norms_query`, `query_transfer`, `W_score` and the + NBFNet `conv` layers present when the checkpoints were trained) are retained + for strict loading; the official evaluation drops them with `strict=False`. +- Prompt graphs follow the released processing (`enclosing=False`: up to 50 + further entities), not the paper's `dist(x,u) + dist(x,v) <= k` alone. +- The reasoner sums messages (the paper writes mean pooling) and updates + entities with a GRU; unreached entities score zero. +- Labels of inverse-query prompts keep the direct example's `(u, v)` order; only + the head and tail tokens are swapped. +- The official sigmoid attention (`attn_type='Sigmoid'`); the GAT option, which + indexes its normalizer by a node ID where an edge ID is meant, is unused. + +## Verification + +Fixtures are generated by `tests/fixtures/kgicl/generate_reference.py` with the +pinned official code plus the patch. It verifies the revision and that tracked +files are unmodified, processes a fixture graph with the official prompt +extraction (hubs, cycles, parallel facts, self-loop facts and examples, +relations with one to five facts, a relation without facts, an isolated entity), +loads it with the official data loader and records official outputs on CPU. +`tiny` uses a small random model (dimension 8, three reasoner layers), `official` +the released `KG-ICL-6L` weights. DICE replays the recorded prompt graphs: + +| Check (float32, CPU) | `tiny` | `official` | +|---|---|---| +| Prompt extraction of every relation's examples | identical entities, facts and tokens | (same graph) | +| Prompt encodings, maximum absolute error | 9.5e-7 | 1.2e-6 | +| All-entity scores of 16 queries, with and without the distance mask | 1.8e-7 | 5.3e-6 (scores up to 15.8) | +| Gradients of every parameter (110 and 184 tensors), evaluation mode | within 3% of `atol = rtol = 1e-4` | within 82% of `atol = rtol = 1e-4` (gradients up to 454) | +| Answer-distance rates | identical | identical | + +The tests use `atol = 2e-5, rtol = 1e-4` for prompts and scores. Prompt sampling +is checked separately: seeded, independent of relation and call order, and +identical after reattaching the graph or reloading an experiment. + +On real graphs, the official evaluation of five of the paper's zero-shot datasets +ran with the patched official code (separate environment: PyTorch 2.7.1, +torch-scatter 2.1.2) and with DICE replaying the prompt graphs the patched official +loader assembles, both on CUDA in float32 with TF32 disabled. DICE used the fused +Triton path. Metrics use the official ranking (ties by entity ID) unless noted: + +| Dataset | Queries × entities | Max. abs. score error | MRR official / DICE | Hits@10 official / DICE | Pessimistic MRR official / DICE | Answers ranked differently | Paper MRR / Hits@10 | +|---|---|---:|---:|---:|---:|---:|---:| +| WN V1 | 352 × 922 | 5.2e-5 | 0.73456 / 0.73455 | 0.8165 / 0.8165 | 0.73350 / 0.73349 | 2 of 376 | 0.733 / 0.838 | +| FB V2 | 791 × 1660 | 6.8e-5 | 0.56350 / 0.56342 | 0.7542 / 0.7542 | 0.56247 / 0.56217 | 5 of 956 | 0.565 / 0.749 | +| NELL V2 | 686 × 2086 | 7.7e-5 | 0.63907 / 0.63907 | 0.8330 / 0.8330 | 0.63605 / 0.63605 | 2 of 952 | 0.644 / 0.835 | +| NL-0 | 1037 × 2026 | 7.2e-5 | 0.56551 / 0.56549 | 0.7870 / 0.7870 | 0.55995 / 0.55990 | 2 of 1526 | 0.557 / 0.777 | +| FB-25 | 6656 × 4097 | 1.1e-4 | 0.42134 / 0.42134 | 0.6873 / 0.6873 | 0.42128 / 0.42123 | 22 of 11432 | 0.396 / 0.656 | + +All scores agree within `atol = rtol = 2e-4`. The differing ranks are near-ties +(score gaps around `1e-5`) that are exact ties in one implementation only. The +official code is not bitwise reproducible across query orders either (CUDA +scatter atomics): its MRR varied by up to 9e-5 (2e-3 on NL-0, which has many +exact ties) over three query orders, a range that contains DICE's values. The +released official code (all five defects) averaged 0.7372, 0.5658, 0.6351, +0.5604 and 0.4191 MRR over three runs, also close to the paper. FB-25 is about +0.025 above the paper with both implementations and the released code. + +```bash +# Offline fixtures and DICE integration; no upstream packages required. +python -m pytest tests/test_kgicl.py -q + +# Include the official KG-ICL-6L checkpoint, verified by SHA-256. +KGICL_CHECKPOINT_DIR=checkpoints/kgicl/KG-ICL-6L python -m pytest tests/test_kgicl.py -q +``` + +To regenerate the fixtures, create an isolated environment with PyTorch, a +matching torch-scatter build, NumPy, SciPy and NetworkX (the fixtures used +torch 2.7.1 and torch-scatter 2.1.2), check out the pinned commit, then run: + +```bash +CUDA_VISIBLE_DEVICES='' python tests/fixtures/kgicl/generate_reference.py \ + /path/to/KG-ICL tests/fixtures/kgicl --checkpoint-dir /path/to/KG-ICL/checkpoint/KG-ICL-6L +``` + +## Complex query answering + +`kgicl-adapter` evaluates KG-ICL as a frozen backbone in the query benchmarks +(`python -m benchmarks.cqa`), with the adapter, beam search and controls of the +ULTRA and TRIX recipes. Prompt graphs come from each benchmark's inference +graph with prompt seed 0 (a recipe may set `prompt_seed`); no distance mask is +applied. Score adapters bind to the backbone through the checkpoint's state +fingerprint, and cached training rows record the prompt settings. The +evaluator caches atomic rows across beams and controls; KG-ICL rows are +bitwise identical whatever the batching or call order, which the tests check +under deterministic algorithms on CPU and CUDA. + +The recipes `benchmarks/plus_h/kgfm_kgicl.json` and +`benchmarks/ultraquery/kgfm_kgicl.json` use the adapter +`benchmarks/adapters/kgicl_product_intersections.json`, fitted with the shared +source recipe of ULTRA and TRIX (training seed 2026090851) and paired with an +identity control. +They cache the per-layer relation +tables of every query relation (`projection_cache_mb` 512, about 260 MiB on +FB15k237); the default 64 MiB recomputes them on graphs with hundreds of +relations, about 2.5 times slower. Outside the benchmark's Docker image, set +`CUBLAS_WORKSPACE_CONFIG=:4096:8`: the evaluator enables deterministic +algorithms, and the prompt encoder uses cuBLAS. + +## License + +The official KG-ICL code and checkpoints are distributed under GPL-3.0 by their +authors; download the checkpoints from the official repository. DICE's +implementation is an independent reimplementation that loads the released state +dictionaries. The bug-fix patch modifies GPL-3.0 code and is distributed under +GPL-3.0. diff --git a/docs/ultra.md b/docs/ultra.md index e584a68a6..773ee0835 100644 --- a/docs/ultra.md +++ b/docs/ultra.md @@ -61,6 +61,13 @@ For direct model use, import `ULTRA` from `dicee.models`, load weights with `load_pretrained(path)`, and attach indexed training triples with `set_graph(...)`. Triples follow DICE's `(head, relation, tail)` order. +Like upstream link prediction, ULTRA conditions the relation graph of a tail query +`(h, r, ?)` on the direct relation, also when `r` is an inverse relation. +`ultra_relation_conditioning="query"` conditions it on `r` itself, as UltraQuery's +projection does; with UltraQuery's weights the model then scores atoms exactly +like UltraQuery's single-source projection. It applies to tail queries +(`forward_k_vs_all`, `forward_k_vs_sample`). + ## Tests ```bash diff --git a/pyproject.toml b/pyproject.toml index 7399ab693..d951ac15f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,12 +57,13 @@ ignore = [ "__init__.py" = ["F401"] # Allow unused imports in __init__.py files [tool.ruff.lint.isort] -known-first-party = ["dicee"] +known-first-party = ["dicee", "benchmarks"] section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"] [tool.pytest.ini_options] # Pytest configuration -testpaths = ["tests"] +testpaths = ["tests", "benchmarks"] +pythonpath = ["."] python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] diff --git a/requirements.txt b/requirements.txt index 9243bb4b0..1c0f95f9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,17 @@ # Core dependencies # NOTE: For CPU-only installation (no NVIDIA CUDA dependencies), use: # pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu -numpy==1.26.4 +numpy==1.26.4; python_version < '3.13' +numpy>=2.1,<3; python_version >= '3.13' torch>=2.5.1 lightning>=2.5.0.post0 pandas>=2.1.0 # Optional dependencies polars>=0.16.14 -# PyArrow 26 requires NumPy 2.x; keep compatible with our NumPy 1.26 pin. -pyarrow>=11.0.0,<26 +# PyArrow 26 requires NumPy 2.x; keep compatible with our NumPy 1.26 pin before Python 3.13. +pyarrow>=11.0.0,<26; python_version < '3.13' +pyarrow>=11.0.0; python_version >= '3.13' rdflib>=7.0.0 tiktoken>=0.5.1 pykeen>=1.10.2 diff --git a/setup.py b/setup.py index 8eb14d41d..3dbb74f85 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,8 @@ # Core dependencies (includes torch and lightning) # Use --extra-index-url https://download.pytorch.org/whl/cpu for CPU-only installation _core_deps = [ - "numpy==1.26.4", + "numpy==1.26.4; python_version < '3.13'", + "numpy>=2.1,<3; python_version >= '3.13'", "torch>=2.5.1", "lightning>=2.5.0.post0", "pandas>=2.1.0", @@ -25,8 +26,9 @@ # Optional dependencies for various features _optional_deps = [ - # PyArrow 26 requires NumPy 2.x; keep compatible with our NumPy 1.26 pin. - "pyarrow>=11.0.0,<26", + # PyArrow 26 requires NumPy 2.x; keep compatible with our NumPy 1.26 pin before Python 3.13. + "pyarrow>=11.0.0,<26; python_version < '3.13'", + "pyarrow>=11.0.0; python_version >= '3.13'", "rdflib>=7.0.0", "tiktoken>=0.5.1", "pykeen>=1.10.2", diff --git a/tests/fixtures/kgicl/generate_reference.py b/tests/fixtures/kgicl/generate_reference.py new file mode 100644 index 000000000..99e3a7702 --- /dev/null +++ b/tests/fixtures/kgicl/generate_reference.py @@ -0,0 +1,192 @@ +"""Generate KG-ICL fixtures with the pinned official code plus upstream-fixes.patch. + +Use an isolated environment with torch, torch-scatter 2.1.2, numpy, scipy and +networkx (the reference environment used torch 2.7.1, numpy 1.24.0). No DICE +code is imported. The generator checks the checkout revision and that its +tracked files are unmodified, applies the committed patch to a temporary copy, +processes a fixture graph with the official prompt extraction, loads it with the +official data loader and records the official model's outputs on CPU: + + CUDA_VISIBLE_DEVICES='' python tests/fixtures/kgicl/generate_reference.py \ + /path/to/KG-ICL tests/fixtures/kgicl +""" +import argparse +import hashlib +import os +import random +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import torch + +COMMIT = "6a3166e347ae468acdfb30a70a2cf3608b66b8f1" +HERE = Path(__file__).resolve().parent + + +def patched_copy(root, workdir): + """Pinned sources with upstream-fixes.patch applied, importable from ``workdir``.""" + head = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip() + assert head == COMMIT, head + assert not subprocess.check_output(["git", "-C", str(root), "diff", "HEAD", "--", "src", "run_your_own_dataset"], text=True) + for name in ("src", "run_your_own_dataset"): + shutil.copytree(root / name, workdir / name) + subprocess.run(["patch", "-p1", "-s", "-d", str(workdir), "-i", str(HERE / "upstream-fixes.patch")], check=True) + sys.path[:0] = [str(workdir / "src"), str(workdir / "run_your_own_dataset")] + + +def fixture_graph(): + """A small graph with hubs, cycles, parallel facts, self-loops and rare relations.""" + rng = random.Random(7) + facts = set() + while len(facts) < 90: + h, t = rng.randrange(30), rng.randrange(30) + if h != t: + facts.add((h, rng.choice((0, 0, 0, 3)), t)) + facts |= {(30, 1, 31), (31, 1, 32), (2, 1, 30), # three facts: fewer cases than shots + (33, 2, 34), # one fact + (5, 4, 6), (6, 4, 33), # two facts + (7, 3, 7), (8, 3, 8), (5, 0, 6), (5, 3, 6), # self-loops and parallel facts + (34, 0, 35), (35, 0, 36), (36, 3, 37), (37, 0, 38)} # a chain away from the core + train = sorted(facts) + valid = [(0, 0, 4), (9, 3, 12), (30, 1, 2), (33, 0, 35), (14, 0, 21), (38, 0, 34), (6, 4, 5), (19, 0, 3)] + test = [(1, 0, 2), (31, 1, 33), (33, 2, 6), (7, 3, 9), (6, 5, 10), (36, 0, 39), (12, 4, 30), (38, 3, 36)] + return train, valid, test, 41, 6 # entity 40 never occurs; relation 5 has no training facts + + +def write_dataset(utils_all, train, valid, test, num_entities, num_relations, destination): + """The official transductive layout: cases from training facts, test background = train + valid.""" + np.random.seed(0) + random.seed(0) + kg = utils_all.KG(train, num_entities, num_relations) + cases = kg.build_cases_for_large_graph(case_num=25, enclosing=False, hop=3) + for split in ("train", "valid", "test"): + directory = destination / split + directory.mkdir(parents=True) + background = train + valid if split == "test" else train + utils_all.write_triple(directory / "background.txt", background) + utils_all.write_triple(directory / "facts.txt", {"train": train, "valid": valid, "test": test}[split]) + utils_all.write_triple(directory / "filter.txt", train + valid + (test if split == "test" else [])) + utils_all.write_dict(directory / "entity2id.txt", {f"e{i}": i for i in range(num_entities)}) + utils_all.write_dict(directory / "relation2id.txt", {f"r{i}": i for i in range(num_relations)}) + for relation in range(num_relations): + (directory / "cases" / f"r{relation}").mkdir(parents=True) + for i, case in enumerate(cases[relation]): + utils_all.write_cases(str(directory / "cases" / f"r{relation}" / str(i)), case) + extracted = {} + for relation, items in cases.items(): + extracted[relation] = [] + for local_triples, labels, entities in items: + order = sorted(entities) + example = local_triples[0] + extracted[relation].append(dict( + example=(entities[example[0]], entities[example[2]]), entities=[entities[k] for k in order], + labels=[list(map(int, labels[k])) for k in order], + triples=sorted({(entities[h], int(r), entities[t]) for h, r, t in local_triples}))) + return extracted + + +def settings(dim, attn_dim, layers, prompt_layers): + return SimpleNamespace( + finetune=False, train_batch_size=8, test_batch_size=64, shot=5, device=torch.device("cpu"), + use_attn=True, attn_type="Sigmoid", AGG="max", AGG_rel="max", MSG="concat", use_augment=False, + use_token_set=True, use_prompt_graph=True, prompt_graph_type="all", path_hop=3, hidden_dim=dim, + attn_dim=attn_dim, n_relation_encoder_layer=prompt_layers, n_layer=layers, act="idd", dropout=0.0, + relation_mask_rate=0.0, use_rspmm=False) + + +def slot_prompts(loader, relation, shot): + """Per-shot prompt graphs exactly as the patched get_case_graph assembles them.""" + kg = loader.kg + edge_index, edge_type, heads, tails, _, _, labels, _ = kg.get_case_graph([relation] * shot, False) + graphs, node_start, edge_start = [], 0, 0 + for slot in range(shot): + case_edges, _, _, _, nodes = kg.case_select(relation, id=slot) + edges = case_edges.shape[1] + graphs.append(dict(edge_index=edge_index[:, edge_start:edge_start + edges] - node_start, + edge_type=edge_type[edge_start:edge_start + edges] - slot * kg.relation_num, + labels=labels[node_start:node_start + nodes], + head=int(heads[slot]) - node_start, tail=int(tails[slot]) - node_start)) + node_start, edge_start = node_start + nodes, edge_start + edges + return graphs + + +def generate(root, destination, checkpoint_dir=None): + torch.set_num_threads(2) + workdir = Path(tempfile.mkdtemp(prefix="kgicl-")) + # The official loader derives split paths by replacing these words. + assert "train" not in str(workdir) and "test" not in str(workdir) and "valid" not in str(workdir) + try: + patched_copy(root, workdir) + import utils_all + from data_loader import DataLoader + from encoder.EntityEncoder import EntityEncoder + + train, valid, test, num_entities, num_relations = fixture_graph() + data = workdir / "fixture" + extracted = write_dataset(utils_all, train, valid, test, num_entities, num_relations, data) + queries = torch.tensor([[0, 0], [5, 3], [30, 1], [33, 2], [5, 4], [7, 3], [6, 5], [40, 0], + [2, 6], [31, 7], [34, 8], [6, 10], [10, 11], [33, 9], [36, 6], [35, 0]]) + variants = [("tiny", 8, 3, 3, 2), ("official", 32, 5, 6, 3)] + for name, dim, attn_dim, layers, prompt_layers in variants: + args = settings(dim, attn_dim, layers, prompt_layers) + loader = DataLoader(args, str(data / "test") + "/", "fixture") + torch.manual_seed(42) + model = EntityEncoder(args) + fixture = dict(upstream_commit=COMMIT, patch_sha256=hashlib.sha256((HERE / "upstream-fixes.patch").read_bytes()).hexdigest(), + num_entities=num_entities, num_relations=num_relations, dim=dim, attn_dim=attn_dim, + num_layers=layers, prompt_layers=prompt_layers, shots=args.shot, + graph=torch.tensor(sorted(map(tuple, loader.kg.background[loader.kg.background[:, 1] < num_relations].tolist()))), + train=torch.tensor(train), valid=torch.tensor(valid), queries=queries, + answer_distance=[float(loader.kg.answer_distance[i]) for i in range(10)], cases=extracted) + if name == "tiny": + with torch.no_grad(): + for parameter in model.parameters(): + parameter.add_(0.3 * torch.randn_like(parameter)) + fixture["state_dict"] = {k: v.clone() for k, v in model.state_dict().items()} + else: + if checkpoint_dir is None: + continue + path = Path(checkpoint_dir) / "model_best.tar" + state = torch.load(path, map_location="cpu", weights_only=True)["state_dict"] + model.load_state_dict({k: v for k, v in state.items() if k in model.state_dict()}, strict=True) + fixture["checkpoint_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + model.eval() + relations = sorted(set(queries[:, 1].tolist())) + fixture["prompts"] = {q: slot_prompts(loader, q, args.shot) for q in relations} + with torch.no_grad(): + fixture["prompt_outputs"] = {} + for q in relations: + batch = loader.get_case_graph(np.array([q] * args.shot), False) + output = model.relation_encoder(*batch, loader, shot=args.shot)[0] + fixture["prompt_outputs"][q] = output[0] + subs, rels = queries[:, 0].numpy(), queries[:, 1].numpy() + fixture["scores"] = model(subs, rels, loader=loader, training=False)[0] + single = torch.cat([model(subs[i:i + 1], rels[i:i + 1], loader=loader, training=False)[0] + for i in range(len(subs))]) + torch.testing.assert_close(single, fixture["scores"], atol=1e-5, rtol=1e-5) + loader.kg.answer_distance = {i: 1.0 for i in range(10)} + fixture["unmasked_scores"] = model(subs, rels, loader=loader, training=False)[0] + # Evaluation-mode gradients are deterministic after fix B1. + weights = torch.randn(fixture["unmasked_scores"].shape, generator=torch.Generator().manual_seed(3)) + (model(subs, rels, loader=loader, training=False)[0] * weights).sum().backward() + fixture["gradient_weights"] = weights + fixture["gradients"] = {k: (None if p.grad is None else p.grad.clone()) for k, p in model.named_parameters()} + torch.save(fixture, destination / f"{name}.pt") + print(name, "generated", flush=True) + finally: + shutil.rmtree(workdir, ignore_errors=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("upstream", type=Path, help="Pinned nju-websoft/KG-ICL checkout") + parser.add_argument("destination", type=Path) + parser.add_argument("--checkpoint-dir", type=Path, help="Directory with the official KG-ICL-6L model_best.tar") + options = parser.parse_args() + os.environ.setdefault("CUDA_VISIBLE_DEVICES", "") + generate(options.upstream.resolve(), options.destination.resolve(), options.checkpoint_dir) diff --git a/tests/fixtures/kgicl/official.pt b/tests/fixtures/kgicl/official.pt new file mode 100644 index 000000000..e5d062e21 Binary files /dev/null and b/tests/fixtures/kgicl/official.pt differ diff --git a/tests/fixtures/kgicl/tiny.pt b/tests/fixtures/kgicl/tiny.pt new file mode 100644 index 000000000..a07f43af4 Binary files /dev/null and b/tests/fixtures/kgicl/tiny.pt differ diff --git a/tests/fixtures/kgicl/upstream-fixes.patch b/tests/fixtures/kgicl/upstream-fixes.patch new file mode 100644 index 000000000..d1e6cda3a --- /dev/null +++ b/tests/fixtures/kgicl/upstream-fixes.patch @@ -0,0 +1,77 @@ +Fixes to nju-websoft/KG-ICL at 6a3166e347ae468acdfb30a70a2cf3608b66b8f1; apply with +patch -p1 in the checkout root. It changes only the defects documented in docs/kgicl.md: + +B1 src/encoder/EntityEncoder.py: the attention activation was a fresh nn.RReLU(), which is + always in training mode, so inference sampled random negative slopes. Use the module's + training flag: random slopes in training, the mean slope in evaluation. +B2 src/data_loader.py: evaluation slot i of the batch-wide relation list selected case i mod + n_cases, which depends on the relation's rank among the batch relations when it has fewer + than shot cases. Select case (i mod shot) mod n_cases. +B3 run_your_own_dataset/utils_all.py: distance labels were paired with prompt nodes in set + iteration order and the loader reads label rows as local node IDs, so most entities got + another entity's tokens. Pair each node with its own label, rows in local-ID order. +B4 run_your_own_dataset/utils_all.py: a self-loop pair was looked up in both directions, + duplicating its facts in the prompt graph. +B5 src/data_loader.py: for a self-loop example fact (u, q, u) the tail token went to local + node 1, an unrelated entity. Both tokens go to the example entity. + +--- a/run_your_own_dataset/utils_all.py ++++ b/run_your_own_dataset/utils_all.py +@@ -265,9 +265,10 @@ + edges_type += list(rels) + edges_index += [(edge[0], edge[1]) for i in range(len(rels))] + # 添加反向边 +- rels = self.ht2r[(edge[1], edge[0])] +- edges_type += list(rels) +- edges_index += [(edge[1], edge[0]) for i in range(len(rels))] ++ if edge[0] != edge[1]: ++ rels = self.ht2r[(edge[1], edge[0])] ++ edges_type += list(rels) ++ edges_index += [(edge[1], edge[0]) for i in range(len(rels))] + edges_index = np.array(edges_index).T + edges_type = np.array(edges_type) + +@@ -279,7 +280,8 @@ + for i in range(len(edges_type)): + if [nodes2id[edges_index[0][i]], edges_type[i], nodes2id[edges_index[1][i]]] != [nodes2id[triple[0]], triple[1], nodes2id[triple[2]]]: + triples.append([nodes2id[edges_index[0][i]], edges_type[i], nodes2id[edges_index[1][i]]]) +- ent2label = {nodes2id[node]: labels[i] for i, node in enumerate(set(enclosing_subgraph_nodes).union({head, tail}))} ++ node_labels = dict(zip(enclosing_subgraph_nodes, labels)) ++ ent2label = {nodes2id[node]: node_labels[node] for node in sorted(nodes2id, key=nodes2id.get)} + ent_subid2ent_id = {nodes2id[node]: node for i, node in enumerate(set(enclosing_subgraph_nodes).union({head, tail}))} + return triples, ent2label, ent_subid2ent_id + +--- a/src/data_loader.py ++++ b/src/data_loader.py +@@ -295,7 +295,7 @@ + num_ent = 0 + for i, rel in enumerate(rels): + if not random_order: +- id_ = i ++ id_ = i % self.args.shot + else: + id_ = None + edges_index, edges_type, query_relation, labels, num_nodes = self.case_select(rel, id=id_) +@@ -305,7 +305,10 @@ + batch_edge_type.append(torch.from_numpy(edges_type + i * (relation_num)).to(self.args.device)) + batch_edge_query_relations.append( + torch.ones(edges_type.shape).to(self.args.device) * query_relation + i * (self.relation_num)) +- if rel < self.relation_num // 2: ++ if edges_index[0][0] == edges_index[1][0]: # self-loop example fact ++ batch_h_positions.append(num_ent) ++ batch_t_positions.append(num_ent) ++ elif rel < self.relation_num // 2: + batch_h_positions.append(num_ent) + batch_t_positions.append(num_ent + 1) + else: +--- a/src/encoder/EntityEncoder.py ++++ b/src/encoder/EntityEncoder.py +@@ -44,7 +44,7 @@ + h_qr = rel_embeddings[q_rel + torch.arange(q_rel.size(0)).to(self.args.device) * (loader.kg.relation_num + 1)][r_idx] + + alpha = self.w_alpha( +- nn.RReLU()(self.Ws_attn(hs) + self.Wr_attn(hr) + self.Wqr_attn(h_qr))) ++ nn.functional.rrelu(self.Ws_attn(hs) + self.Wr_attn(hr) + self.Wqr_attn(h_qr), training=self.training)) + alpha = torch.sigmoid(alpha) # [N_edge_of_all_batch, 1] + + # if not torch.cuda.is_available(): diff --git a/tests/fixtures/query_baselines/README.md b/tests/fixtures/query_baselines/README.md new file mode 100644 index 000000000..bbd9dd901 --- /dev/null +++ b/tests/fixtures/query_baselines/README.md @@ -0,0 +1,66 @@ +# Upstream inference fixtures + +Generated on CPU with PyTorch 2.9.1. No DICE imports occur in the generator. +Each file records its upstream git commit, graph, queries, checkpoint state, +options, and expected complete score vectors. `ultra.pt` also records hashes of +the public `ultraquery.pth` and `ultra_3g.pth` weights. The random seed is 20260926. + +The repositories and immutable commits are listed in `REFERENCES` in +`dicee/query_answering/catalog.py`. +Clone each repository and check out that commit before regenerating a fixture. + +```bash +PYTHONPATH=/path/to/upstream \ +python tests/fixtures/query_baselines/generate_reference.py \ + cone /path/to/upstream tests/fixtures/query_baselines/cone.pt +``` + +Use `cqd`, `clmpt`, `ultra`, `gnnqe`, or `qto` for the other fixtures. For QTO, +put its `kbc` directory on `PYTHONPATH`. Use separate processes to avoid upstream +package-name collisions. Generating ULTRA/GNN-QE requires their reference-only +dependencies; these are not DICE runtime dependencies. + +The verification environment used torch-geometric 2.6.1, torch-scatter 2.1.2, +torch-cluster 1.6.3, TorchDrug 0.2.1, and Ninja. GPU visibility was disabled and +`MAX_JOBS=1` bounded reference extension compilation. For current Python/RDKit, +the generator aliases `collections.Sequence` and supplies the unused drawing +base class removed from RDKit. TorchDrug requires these two build-only changes +with PyTorch 2.9: + +1. In `layers/functional/extension/{spmm,rspmm}.h`, change + `ATen/SparseTensorUtils.h` to `ATen/native/SparseTensorUtils.h`. +2. In `utils/torch.py`, pass `extra_ldflags`, `extra_include_paths`, + `build_directory`, and `verbose` to `cpp_extension.load` by keyword. + +The original sparse kernels and neural computations are unchanged. GNN-QE +fixtures execute those compiled kernels, including symbolic traversal. + +CLMPT fixtures evaluate DNF branches through the released reasoner with the +original relation groundings. Both pre/post-norm variants are covered. The +metadata's earlier incorrect `up` repair note has been corrected by regenerating +the fixture; all tensors remain bit-identical. +CQD fixtures intentionally retain the released 3p/4p quirks. +An additional plain CQD case verifies the `ip`/`up` cap with beam 6 and max_k 1. +CLMPT also records paired-query batches to test the post-norm layout directly. + +The fixtures cover all 16 shapes for ConE, QTO, CLMPT, UltraQuery and GNN-QE; +CQD/CQD-Hybrid cover 11 positive shapes with product and min in `cqd.pt`, plus +the five negated shapes in `cqd-negation.pt`. + +`cqd-negation.pt` covers the five negated types. +Its executor is official CQD-A at commit +`642ce042708be6247087c09c780b9deb47e941d3`, with +H min-max atomic scoring. +Both operators are covered. Hybrid checks use a full-width beam and observed +fact overrides; its dynamic beam is covered separately by the positive fixtures. + +```bash +python tests/fixtures/query_baselines/generate_negation_reference.py \ + /path/to/adaptive-cqd tests/fixtures/query_baselines/cqd-negation.pt +``` + +For ConE, CLMPT, GNN-QE, CQD or QTO, `--checkpoint /path/to/released-weights` generates +an additional reference using those learned parameters. Vocabulary tensors +are cropped to this seven-entity, four-relation graph; other learned parameters +remain unchanged. The output records the original checkpoint hash. Verify full +checkpoint loading against its original dataset separately. Store these large +references under `Experiments/`, outside the checked-in fixtures. diff --git a/tests/fixtures/query_baselines/clmpt.pt b/tests/fixtures/query_baselines/clmpt.pt new file mode 100644 index 000000000..f3a95688c Binary files /dev/null and b/tests/fixtures/query_baselines/clmpt.pt differ diff --git a/tests/fixtures/query_baselines/cone.pt b/tests/fixtures/query_baselines/cone.pt new file mode 100644 index 000000000..b2da1c4bc Binary files /dev/null and b/tests/fixtures/query_baselines/cone.pt differ diff --git a/tests/fixtures/query_baselines/cqd-negation.pt b/tests/fixtures/query_baselines/cqd-negation.pt new file mode 100644 index 000000000..d73c7e9f8 Binary files /dev/null and b/tests/fixtures/query_baselines/cqd-negation.pt differ diff --git a/tests/fixtures/query_baselines/cqd.pt b/tests/fixtures/query_baselines/cqd.pt new file mode 100644 index 000000000..b19e967ac Binary files /dev/null and b/tests/fixtures/query_baselines/cqd.pt differ diff --git a/tests/fixtures/query_baselines/generate_negation_reference.py b/tests/fixtures/query_baselines/generate_negation_reference.py new file mode 100644 index 000000000..f30274522 --- /dev/null +++ b/tests/fixtures/query_baselines/generate_negation_reference.py @@ -0,0 +1,74 @@ +"""CQD-A negation oracles with +H atomic scoring; no DICE imports.""" + +import argparse +import hashlib +import subprocess +from pathlib import Path + +import torch +from generate_reference import flatten, module + +COMMIT = '642ce042708be6247087c09c780b9deb47e941d3' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('upstream', type=Path) + parser.add_argument('output', type=Path) + args = parser.parse_args() + commit = subprocess.check_output(['git', '-C', str(args.upstream), 'rev-parse', 'HEAD'], text=True).strip() + if commit != COMMIT: + raise ValueError('Unexpected CQD-A source revision') + source = module(args.upstream / 'cqda/cqd/discrete.py', 'cqda_reference') + base = Path(__file__).with_name('cqd.pt') + fixture = torch.load(base, weights_only=True) + queries = {name: value for name, value in fixture['queries'].items() if 'n' in name} + cases = [] + torch.set_num_threads(2) + with torch.no_grad(): + for parent in fixture['cases']: + if '1p' not in parent['scores']: + continue + state, hybrid = parent['state'], parent['name'] == 'cqd-hybrid' + entity = torch.nn.Embedding.from_pretrained(state['embeddings.0.weight']) + relation = torch.nn.Embedding.from_pretrained(state['embeddings.1.weight']) + norm = parent['config']['tnorm'] + k = fixture['num_entities'] if hybrid else 2 + + def scoring(lhs, rel, rhs): + hr, hi = lhs.chunk(2, -1) + rr, ri = rel.chunk(2, -1) + tr, ti = rhs.chunk(2, -1) + raw = (hr * rr - hi * ri) @ tr.T + (hr * ri + hi * rr) @ ti.T + result = (.9 if hybrid else 1.) * (raw - raw.min()) / (raw.max() - raw.min()) + if hybrid: + heads = (lhs[:, None] == entity.weight[None]).all(-1).int().argmax(-1) + rels = (rel[:, None] == relation.weight[None]).all(-1).int().argmax(-1) + for row, (h, r) in enumerate(zip(heads.tolist(), rels.tolist())): + for a, b, t in fixture['triples']: + if (a, b) == (h, r): + result[row, t] = 1 + return result + + def negation(value, **kwargs): + return 1 - value + + scores = {} + for name, query in queries.items(): + options = {'aim_run': None} if name in ('2in', '3in') else {'k': k} + scores[name] = getattr(source, f'query_{name}')( + entity, relation, torch.tensor([flatten(query)]), scoring, + t_norm=torch.mul if norm == 'prod' else torch.minimum, negation=negation, **options)[0] + cases.append(dict(name=parent['name'], state=state, scores=scores, + config=dict(atomic_negation=True, beam_size=k, max_k=k, tnorm=norm))) + output = {key: fixture[key] for key in ('triples', 'num_entities', 'num_relations')} + output.update(queries=queries, cases=cases, upstream_commit=commit, + source='https://github.com/EdinburghNLP/adaptive-cqd', + parent_sha256=hashlib.sha256(base.read_bytes()).hexdigest(), + scope='CQD-A signed-atom executor with +H calibration; hybrid full-width observed override.') + torch.save(output, args.output) + print(f'{len(cases)} cases, {sum(len(c["scores"]) for c in cases)} negation vectors') + + +if __name__ == '__main__': + main() diff --git a/tests/fixtures/query_baselines/generate_reference.py b/tests/fixtures/query_baselines/generate_reference.py new file mode 100644 index 000000000..8c44626a5 --- /dev/null +++ b/tests/fixtures/query_baselines/generate_reference.py @@ -0,0 +1,301 @@ +"""Generate CPU oracles from pinned upstream code; never import DICE here. + +Run each method in its own process with its repository on PYTHONPATH. +Optional reference dependencies belong in an isolated environment. +""" + +import argparse +import hashlib +import importlib.util +import subprocess +from pathlib import Path + +import torch + +A, B, C, D = (0, (0,)), (1, (2,)), (2, (1,)), (3, (3,)) +QUERIES = { + '1p': A, '2p': (0, (0, 2)), '3p': (0, (0, 2, 1)), '4p': (0, (0, 2, 1, 3)), + '2i': (A, B), '3i': (A, B, C), '4i': (A, B, C, D), + 'ip': ((A, B), (2,)), 'pi': ((0, (0, 2)), B), + '2u': (A, B, (-1,)), 'up': ((A, B, (-1,)), (2,)), + '2in': (A, (1, (2, -2))), '3in': (A, C, (1, (2, -2))), + 'inp': ((A, (1, (2, -2))), (2,)), 'pin': ((0, (0, 2)), (1, (2, -2))), + 'pni': ((0, (0, 2, -2)), B), +} +DIRECT = [(0, 0, 1), (0, 0, 2), (1, 2, 3), (2, 2, 4), (3, 0, 5), (5, 2, 6), (4, 0, 0)] +TRIPLES = sorted({*DIRECT, *((t, r ^ 1, h) for h, r, t in DIRECT)}) +N, R = 7, 4 + + +def flatten(query): + return [value for child in query for value in flatten(child)] if isinstance(query, tuple) else [query] + + +def shape(query): + if isinstance(query[0], int) and len(query) == 2: + return 'e', tuple('n' if r == -2 else 'r' for r in query[1]) + if len(query) == 2 and all(isinstance(x, int) for x in query[1]): + return shape(query[0]), tuple('n' if r == -2 else 'r' for r in query[1]) + return tuple(('u',) if child == (-1,) else shape(child) for child in query) + + +def module(path, name): + spec = importlib.util.spec_from_file_location(name, path) + result = importlib.util.module_from_spec(spec) + spec.loader.exec_module(result) + return result + + +def complex_checkpoint(path): + if path is None: + return None + state = torch.load(path, map_location='cpu', weights_only=True) + while any(key in state for key in ('model', 'model_state_dict', 'state_dict')): + state = next(state[key] for key in ('model', 'model_state_dict', 'state_dict') if key in state) + if set(state) != {'embeddings.0.weight', 'embeddings.1.weight'}: + raise ValueError('Expected the released ComplEx embedding tensors') + return {key: value[:N if key == 'embeddings.0.weight' else R].clone() for key, value in state.items()} + + +def cone(root, checkpoint=None): + source = module(root / 'models.py', 'cone_reference') + names = {shape(q): name + '-DNF' if name in ('2u', 'up') else name for name, q in QUERIES.items()} + state = None + if checkpoint: + state = torch.load(checkpoint, map_location='cpu', weights_only=True)['model_state_dict'] + for key, count in (('entity_embedding', N), ('axis_embedding', R), ('arg_embedding', R)): + state[key] = state[key][:count].clone() + dim, gamma = (state['entity_embedding'].shape[1], state['gamma'].item()) if state else (8, 12.) + model = source.KGReasoning(N, R, dim, gamma, query_name_dict=names, center_reg=.02) + model.cone_proj = source.ConeProjection(dim, state['cone_proj.layer1.weight'].shape[0] if state else 16, 2) + if state: + model.load_state_dict(state, strict=True) + model.eval() + scores = {name: model(None, torch.arange(N)[None], None, + {shape(q): torch.tensor([flatten(q)])}, {shape(q): [0]})[1][0] + for name, q in QUERIES.items()} + return [dict(name='cone', state=model.state_dict(), config={'center_reg': .02}, scores=scores)] + + +def cqd(root, checkpoint=None): + from cqd.base import CQD + state = complex_checkpoint(checkpoint) + rank = state['embeddings.0.weight'].shape[1] // 2 if state else 4 + names = {shape(q): name + '-DNF' if name in ('2u', 'up') else name for name, q in QUERIES.items() if 'n' not in name} + filters = {} + for h, r, t in TRIPLES: + filters.setdefault((h, r), []).append(t) + result = [] + for hybrid in (False, True): + for norm in ('prod', 'min'): + model = CQD(N, R, rank, k=2, max_k=4, max_norm=.9 if hybrid else 1., filters=filters, + do_normalize=True, t_norm_name=norm, query_name_dict=names).eval() + if state: + model.load_state_dict(state, strict=True) + else: + for parameter in model.parameters(): + parameter.normal_() + scores = {name: model(None, None, None, {shape(q): torch.tensor([flatten(q)])}, {shape(q): [0]})[1][0] + for name, q in QUERIES.items() if 'n' not in name} + result.append(dict(name='cqd-hybrid' if hybrid else 'cqd', state=model.state_dict(), + config=dict(beam_size=2, max_k=4, tnorm=norm), scores=scores)) + model = CQD(N, R, rank, k=6, max_k=1, max_norm=1., filters=filters, + do_normalize=True, t_norm_name='prod', query_name_dict=names).eval() + model.load_state_dict(result[0]['state'], strict=True) + scores = {name: model(None, None, None, {shape(QUERIES[name]): torch.tensor([flatten(QUERIES[name])])}, + {shape(QUERIES[name]): [0]})[1][0] for name in ('ip', 'up')} + result.append(dict(name='cqd', state=model.state_dict(), + config=dict(beam_size=6, max_k=1, tnorm='prod'), scores=scores)) + return result + + +def qto(root, checkpoint=None): + source = module(root / 'model.py', 'qto_reference') + from src.models import ComplEx + state = complex_checkpoint(checkpoint) + rank = state['embeddings.0.weight'].shape[1] // 2 if state else 4 + backbone = ComplEx([N, R, N], rank=rank, init_size=1.) + if state: + backbone.load_state_dict(state, strict=True) + rows = [] + for relation in range(R): + edges = [(h, t) for h, r, t in TRIPLES if r == relation] + value = source.neural_adj_matrix(backbone, relation, N, 'cpu', .08, edges) + value = (value >= 1).float() * .9999 + (value < 1).float() * value + for h, t in edges: + value[h, t] = 1 + rows.append(value) + model = source.KGReasoning.__new__(source.KGReasoning) + torch.nn.Module.__init__(model) + model.nentity, model.device, model.fraction, model.neg_scale = N, 'cpu', 2, 3. + model.relation_embeddings = [[row[:N // 2].to_sparse(), row[N // 2:].to_sparse()] for row in rows] + scores = {name: model.embed_query(torch.tensor([flatten(q)]), shape(q), 0)[0][0] for name, q in QUERIES.items()} + return [dict(name='qto', state=backbone.state_dict(), config=dict(threshold=.08, negation_scale=3.), scores=scores, + adjacency=torch.stack(rows))] + + +def ultra(root, checkpoint=None): + from torch_geometric.data import Data + from ultra import datasets_query, tasks # noqa: F401 - initialize the upstream circular import + from ultra.models import Ultra + from ultra.query_utils import Query + from ultra.ultraquery import UltraQuery + triples = torch.tensor(TRIPLES) + graph = Data(edge_index=triples[:, [0, 2]].T, edge_type=triples[:, 1], num_nodes=N, num_relations=R) + tasks.build_relation_graph(graph) + result = [] + for checkpoint, threshold in ((None, 0.), ('ultraquery.pth', 0.), ('ultra_3g.pth', .8)): + dim, layers = (8, 2) if checkpoint is None else (64, 6) + config = dict(input_dim=dim, hidden_dims=[dim] * layers, message_func='distmult', aggregate_func='sum', + short_cut=True, layer_norm=True) + backbone = Ultra(dict(config, **{'class': 'RelNBFNet'}), dict(config, **{'class': 'QueryNBFNet'})) + model = UltraQuery(backbone, threshold=threshold).eval() + if checkpoint: + state = torch.load(root / 'ckpts' / checkpoint, weights_only=True, map_location='cpu')['model'] + if checkpoint == 'ultraquery.pth': + model.load_state_dict(state, strict=True) + else: + backbone.load_state_dict(state, strict=True) + scores = {name: model(graph, Query.from_nested(q)[None], symbolic_traversal=False)[0] for name, q in QUERIES.items()} + item = dict(name='ultraquery', state=model.state_dict(), config=dict(threshold=threshold), scores=scores) + if checkpoint: + item['checkpoint'] = checkpoint + item['checkpoint_sha256'] = hashlib.sha256((root / 'ckpts' / checkpoint).read_bytes()).hexdigest() + result.append(item) + return result + + +def clmpt(root, checkpoint=None): + from src.language.foq import EFO1Query + from src.language.grammar import parse_lstr_to_lformula + from src.pipeline.clmpt import CLMPTLayer, CLMPTReasoner + from src.structure.nbp_complex import ComplEx + # Independent literal formulas in the author's query language. + formulas = { + '1p': 'r1(s1,f)', '2p': '(r1(s1,e1))&(r2(e1,f))', + '3p': '((r1(s1,e1))&(r2(e1,e2)))&(r3(e2,f))', + '4p': '(((r1(s1,e1))&(r2(e1,e2)))&(r3(e2,e3)))&(r4(e3,f))', + '2i': '(r1(s1,f))&(r2(s2,f))', '3i': '((r1(s1,f))&(r2(s2,f)))&(r3(s3,f))', + '4i': '(((r1(s1,f))&(r2(s2,f)))&(r3(s3,f)))&(r4(s4,f))', + 'ip': '((r1(s1,e1))&(r2(s2,e1)))&(r3(e1,f))', + 'pi': '((r1(s1,e1))&(r2(e1,f)))&(r3(s2,f))', + '2in': '(r1(s1,f))&(!(r2(s2,f)))', '3in': '((r1(s1,f))&(r2(s2,f)))&(!(r3(s3,f)))', + 'inp': '((r1(s1,e1))&(!(r2(s2,e1))))&(r3(e1,f))', + 'pin': '((r1(s1,e1))&(r2(e1,f)))&(!(r3(s2,f)))', + 'pni': '((r1(s1,e1))&(!(r2(e1,f))))&(r3(s2,f))', + } + values = {'1p': [0, 0], '2p': [0, 0, 2], '3p': [0, 0, 2, 1], '4p': [0, 0, 2, 1, 3], + '2i': [0, 0, 1, 2], '3i': [0, 0, 1, 2, 2, 1], '4i': [0, 0, 1, 2, 2, 1, 3, 3], + 'ip': [0, 0, 1, 2, 2], 'pi': [0, 0, 2, 1, 2], '2in': [0, 0, 1, 2], + '3in': [0, 0, 2, 1, 1, 2], 'inp': [0, 0, 1, 2, 2], 'pin': [0, 0, 2, 1, 2], + 'pni': [0, 0, 2, 1, 2]} + # Grounding order is the atom traversal's first occurrence of each symbol/relation. + import re + state = torch.load(checkpoint, map_location='cpu', weights_only=True) if checkpoint else None + if state: + for key, count in (('nbp._entity_embedding.weight', N), ('nbp._relation_embedding.weight', R)): + state[key] = state[key][:count].clone() + result = [] + modes = [any(key.startswith('transformer.encoder.') for key in state)] if state else (True, False) + for pre_norm in modes: + rank = state['nbp._entity_embedding.weight'].shape[1] // 2 if state else 4 + backbone = ComplEx(N, R, rank, init_size=1.) + hidden_key = 'transformer.encoder.layers.0.ffn.layer1.weight' if pre_norm else 'transformer.layers.0.linear1.weight' + hidden = state[hidden_key].shape[0] if state else 16 + layer = CLMPTLayer(hidden, backbone, layers=2, pre_norm=pre_norm).eval() + if state: + layer.load_state_dict(state, strict=True) + reasoner = CLMPTReasoner(backbone, layer) + + def score(name, override=None, batch_overrides=None): + formula = formulas[name] + symbols = [] + for relation, source, target in re.findall(r'(r\d+)\((\w+),(\w+)\)', formula): + for key in (source, relation): + if key.startswith(('s', 'r')) and key not in symbols: + symbols.append(key) + query = EFO1Query(parse_lstr_to_lformula(formula)) + rows = batch_overrides if batch_overrides is not None else [values[name] if override is None else override] + for row in rows: + query.append_qa_instances(dict(zip(symbols, row)), {'f': []}, {'f': []}) + reasoner.initialize_with_query(query) + reasoner.estimate_variable_embeddings() + scores = torch.cosine_similarity(reasoner.get_ent_emb('f')[:, None], backbone.entity_embedding, dim=-1) + return scores if batch_overrides is not None else scores[0] + + scores = {name: score(name) for name in formulas} + scores['2u'] = torch.maximum(score('1p'), score('1p', [1, 2])) + scores['up'] = torch.maximum(score('2p'), score('2p', [1, 2, 2])) + batches = {'2p': dict(queries=[QUERIES['2p'], (1, (2, 3))], scores=score('2p', batch_overrides=[[0, 0, 2], [1, 2, 3]])), + '2i': dict(queries=[QUERIES['2i'], ((2, (1,)), (3, (3,)))], + scores=score('2i', batch_overrides=[[0, 0, 1, 2], [2, 1, 3, 3]]))} + result.append(dict(name='clmpt', state=layer.state_dict(), config={}, scores=scores, batches=batches, + oracle_note='DNF branches evaluated separately with the released relation groundings')) + return result + + +def gnnqe(root, checkpoint=None): + # Import compatibility only; the neural and sparse inference code is unchanged. + import collections + import collections.abc + import sys + import types + collections.Sequence = collections.abc.Sequence + drawing = types.ModuleType('rdkit.Chem.Draw.mplCanvas') + drawing.Canvas = object + sys.modules[drawing.__name__] = drawing + from gnnqe.data import Query + from gnnqe.gnn import NeuralBellmanFordNetwork + from gnnqe.model import QueryExecutor + from torchdrug import data + triples = torch.tensor(TRIPLES)[:, [0, 2, 1]] + graph = data.Graph(triples, num_node=N, num_relation=R) + state = None + if checkpoint: + with torch.serialization.safe_globals([data.Graph]): + state = torch.load(checkpoint, map_location='cpu', weights_only=True)['model'] + state = {key.removeprefix('model.'): value for key, value in state.items() if key not in ('fact_graph', 'graph')} + dim = state['model.query.weight'].shape[1] + count = len({key.split('.')[3] for key in state if key.startswith('model.model.layers.')}) + state['model.query.weight'] = state['model.query.weight'][:R].clone() + for key in state: + if '.relation_linear.' in key: + state[key] = state[key][:R * dim].clone() + else: + dim, count = 8, 2 + result = [] + for aggregate in (['pna'] if checkpoint else ['sum', 'pna']): + concat_hidden = state is not None and state['model.mlp.layers.0.weight'].shape[1] != 2 * dim + backbone = NeuralBellmanFordNetwork(dim, [dim] * count, R, aggregate_func=aggregate, + short_cut=True, layer_norm=True, concat_hidden=concat_hidden) + model = QueryExecutor(backbone).eval() + if state: + model.load_state_dict(state, strict=True) + scores = {name: model(graph, Query.from_nested(q)[None])[0] for name, q in QUERIES.items()} + result.append(dict(name='gnnqe', state=model.state_dict(), config=dict(aggregate=aggregate), scores=scores)) + return result + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('method', choices=['cone', 'cqd', 'qto', 'ultra', 'clmpt', 'gnnqe']) + parser.add_argument('upstream', type=Path) + parser.add_argument('output', type=Path) + parser.add_argument('--checkpoint', type=Path) + args = parser.parse_args() + torch.set_num_threads(2) + torch.manual_seed(20260926) + with torch.no_grad(): + if args.checkpoint and args.method == 'ultra': + parser.error('ULTRA generation uses the named public checkpoints under upstream/ckpts') + cases = globals()[args.method](args.upstream, args.checkpoint) + commit = subprocess.check_output(['git', '-C', str(args.upstream), 'rev-parse', 'HEAD'], text=True).strip() + args.output.parent.mkdir(parents=True, exist_ok=True) + fixture = dict(upstream_commit=commit, queries=QUERIES, triples=TRIPLES, num_entities=N, num_relations=R, cases=cases) + if args.checkpoint: + with args.checkpoint.open('rb') as stream: + checksum = hashlib.file_digest(stream, 'sha256').hexdigest() + fixture.update(checkpoint_sha256=checksum, checkpoint=str(args.checkpoint), + vocabulary='first 7 entities and 4 relations; learned non-vocabulary parameters unchanged') + torch.save(fixture, args.output) + print(f'{args.method}: {len(cases)} cases, {sum(len(case["scores"]) for case in cases)} score vectors', flush=True) diff --git a/tests/fixtures/query_baselines/gnnqe.pt b/tests/fixtures/query_baselines/gnnqe.pt new file mode 100644 index 000000000..ff7f4dd4a Binary files /dev/null and b/tests/fixtures/query_baselines/gnnqe.pt differ diff --git a/tests/fixtures/query_baselines/qto.pt b/tests/fixtures/query_baselines/qto.pt new file mode 100644 index 000000000..f228a4ca8 Binary files /dev/null and b/tests/fixtures/query_baselines/qto.pt differ diff --git a/tests/fixtures/query_baselines/ultra.pt b/tests/fixtures/query_baselines/ultra.pt new file mode 100644 index 000000000..776436b23 Binary files /dev/null and b/tests/fixtures/query_baselines/ultra.pt differ diff --git a/tests/test_inference_kernels.py b/tests/test_inference_kernels.py new file mode 100644 index 000000000..bc0942584 --- /dev/null +++ b/tests/test_inference_kernels.py @@ -0,0 +1,323 @@ +"""Device-cache isolation, exact kernel scheduling and host-synchronization checks for H100 settings.""" + +import importlib + +import pytest +import torch + +from dicee.models import ULTRA +from dicee.models._inference import cached_relation_batch, float32_precision_backends +from dicee.query_answering.engine import AtomicBatchCache, AtomicScorer + +# Module-level imports: Triton 3.1 resolves the names in a kernel from module globals only, not from an +# enclosing function, so the oracle kernels below fail with NameError('tl is not defined') otherwise. +try: + import triton + import triton.language as tl + from triton.language.extra.cuda import libdevice +except ImportError: # CPU-only installs: every Triton test calls pytest.importorskip('triton'). + triton = tl = libdevice = None + +CUDA = pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA unavailable') +DEVICES = ['cpu', pytest.param('cuda', marks=CUDA)] + + +class TableModel(torch.nn.Module): + def __init__(self, values): + super().__init__() + self.table = torch.nn.Parameter(values.clone()) + self.num_entities, self.num_relations = values.shape[:2] + + def forward_k_vs_all(self, pairs): + return self.table[pairs[:, 0], pairs[:, 1]] + + +@pytest.fixture(autouse=True) +def execution_settings(): + previous = torch.get_num_threads() + # Not monkeypatch: since pytest 9.2 its undo calls delattr on objects without a data descriptor, and the + # precision objects of PyTorch serve fp32_precision through __getattr__/__setattr__. + saved = [(backend, backend.fp32_precision) for backend in float32_precision_backends()] + torch.set_num_threads(2) + try: + for backend, _ in saved: + backend.fp32_precision = 'ieee' + yield + finally: + for backend, precision in reversed(saved): + backend.fp32_precision = precision + torch.set_num_threads(previous) + + +@pytest.mark.parametrize('device', DEVICES) +@pytest.mark.parametrize('storage', ['cpu', 'model']) +def test_atomic_cache_placement_isolation_eviction_and_invalidation(device, storage): + values = torch.arange(50., dtype=torch.float64).reshape(5, 2, 5).to(device) + model = TableModel(values).eval() + cache = AtomicBatchCache(80, device=storage) + scorer = AtomicScorer(model, row_batch_size=2, raw_cache=cache) + pair = [(0, 0), (1, 1)] + expected = scorer.rows(pair) + saved = next(iter(cache.rows.values())) + assert saved.device.type == (device if storage == 'model' else 'cpu') + assert saved.data_ptr() != expected.data_ptr() + expected.zero_() + reused = scorer.rows(pair) + assert torch.equal(reused, values[[0, 1], [0, 1]]) + reused.zero_() + assert torch.equal(scorer.rows(pair), saved.to(device)) + scorer.rows([(2, 0)]) + assert tuple(pair) not in cache.rows and cache.used == 40 and cache.peak == 80 + with torch.no_grad(): + model.table.add_(1) + assert torch.equal(scorer.rows(pair), values[[0, 1], [0, 1]] + 1) + assert list(cache.rows) == [tuple(pair)] + + +def test_zero_budget_and_invalid_cache_device(): + cache = AtomicBatchCache(0, device='model') + cache.get('token', [(0, 0)]) + cache.put([(0, 0)], torch.ones(1, 5)) + assert cache.used == 0 and not cache.rows + with pytest.raises(ValueError, match='device'): + AtomicBatchCache(100, device='cuda') + + +def test_same_relation_batch_shares_storage_and_mixed_batch_preserves_order(): + values = {0: torch.randn(6, 8), 1: torch.randn(6, 8)} + batch = cached_relation_batch(values, [1] * 32) + assert batch.stride(0) == 0 and batch.data_ptr() == values[1].data_ptr() + assert batch.untyped_storage().nbytes() == values[1].numel() * values[1].element_size() + ids = [1, 0, 1, 0] + assert torch.equal(cached_relation_batch(values, ids), torch.stack([values[key] for key in ids])) + + +@pytest.mark.parametrize('device', DEVICES) +def test_broadcast_cached_relations_preserve_scores_order_and_cache(device, monkeypatch): + cls = ULTRA + torch.manual_seed(11) + facts = torch.tensor([[0, 0, 1], [1, 1, 2], [2, 0, 3], [3, 1, 0]]) + prefix = cls.name.lower() + model = cls(dict(num_entities=6, num_relations=2, **{prefix + '_dim': 8}, + ultra_num_layers=2, **{prefix + '_query_batch_size': 4})).set_graph(facts).to(device).eval().requires_grad_(False) + queries = torch.tensor([[0, 0], [1, 0], [2, 0], [4, 0], [1, 1], [3, 0]], device=device) + module = importlib.import_module('dicee.models.' + prefix) + with torch.no_grad(), monkeypatch.context() as patch: + patch.setattr(module, 'cached_relation_batch', lambda values, ids: torch.stack([values[key] for key in ids])) + expected = model(queries) + cached = model._relation_cache if cls is ULTRA else model._initial_cache + before = {key: value.clone() for key, value in cached.items()} + with torch.no_grad(): + actual = model(queries) + torch.testing.assert_close(actual, expected, atol=2e-5, rtol=2e-5) + assert torch.equal(actual.argsort(-1), expected.argsort(-1)) + assert all(torch.equal(cached[key], value) for key, value in before.items()) + + +@CUDA +@pytest.mark.parametrize('dim', [7, 64, 128]) +@pytest.mark.parametrize('reduction', ['sum', 'min', 'max']) +def test_one_warp_ordered_aggregation_is_bitwise_equal_to_four_warps(dim, reduction): + pytest.importorskip('triton') + from dicee.query_answering.methods._ordered_aggregation import layout + from dicee.query_answering.methods._ordered_aggregation_cuda import ordered_reduce + + generator = torch.Generator().manual_seed(17) + # Hub, empty row, repeated edges, non-contiguous states and broadcast relations. + edges = torch.randint(10, (2, 157), generator=generator) + edges[0, :100] = 0 + types = torch.randint(3, (157,), generator=generator) + csr = layout(edges.cuda(), types.cuda(), 11) + states = torch.randn(5, 11, dim * 2, generator=generator).cuda()[..., ::2].transpose(0, 1) + relations = torch.randn(3, 1, dim, generator=generator).cuda().expand(-1, 5, -1) + expected = ordered_reduce(states, relations, csr, reduction, num_warps=4) + actual = ordered_reduce(states, relations, csr, reduction, num_warps=1) + assert torch.equal(actual, expected) + +def _single_pass_rows(): + """The previous one-program-per-row kernel, retained as the bitwise oracle.""" + + @triton.jit + def rows(S, R, B, O, PTR, SRC, TYPE, + SB: tl.constexpr, SN: tl.constexpr, SD: tl.constexpr, + RB: tl.constexpr, RN: tl.constexpr, RD: tl.constexpr, + BB: tl.constexpr, BN: tl.constexpr, BD: tl.constexpr, + N: tl.constexpr, D: tl.constexpr, FEATURES: tl.constexpr, EDGES: tl.constexpr): + row, batch = tl.program_id(0), tl.program_id(1) + feature = tl.arange(0, FEATURES) + start, end = tl.load(PTR + row), tl.load(PTR + row + 1) + total = tl.full((FEATURES,), 0, tl.float32) + for first in range(start, end, EDGES): + edge = first + tl.arange(0, EDGES) + source = tl.load(SRC + edge, edge < end, other=0) + relation = tl.load(TYPE + edge, edge < end, other=0) + mask = (edge[:, None] < end) & (feature[None, :] < D) + state = tl.load(S + batch * SB + source[:, None] * SN + feature[None, :] * SD, mask, other=0) + rel = tl.load(R + batch * RB + relation[:, None] * RN + feature[None, :] * RD, mask, other=0) + total += tl.sum(state.to(tl.float32) * rel.to(tl.float32), axis=0) + boundary = tl.load(B + batch * BB + row * BN + feature * BD, feature < D, other=0) + tl.store(O + (batch * N + row) * D + feature, total + boundary, feature < D) + + def launch(states, boundary, relations, layout): + batch, nodes, dim = states.shape + output = torch.empty_like(states, memory_format=torch.contiguous_format) + rows[(nodes, batch)](states, relations, boundary, output, layout.offsets, layout.sources, layout.types, + *states.stride(), *relations.stride(), *boundary.stride(), + nodes, dim, triton.next_power_of_2(dim), 32, num_warps=4, enable_fp_fusion=False) + return output + return launch + + +def _hub_graph(nodes, edges, types, generator, device): + index = torch.randint(nodes, (2, edges), generator=generator) + index[0, :edges // 3] = 1 # one hub with hundreds of chunks + index[0, edges // 3:edges // 2] = nodes - 1 # a second hub at the last row + index[0, index[0] == 2] = 3 # an empty row + return index.to(device), torch.randint(types, (edges,), generator=generator).to(device) + + +@CUDA +@pytest.mark.parametrize('batch', [1, 3, 5, 16]) +@pytest.mark.parametrize('dim', [7, 32, 64]) +@pytest.mark.parametrize('dtype', [torch.float32, torch.bfloat16]) +def test_hub_split_rows_are_bitwise_equal_to_single_pass_rows(batch, dim, dtype): + pytest.importorskip('triton') + from dicee.models._fused_message import HUB_TILES, csr_layout + from dicee.models._triton_message import distmult_sum + generator = torch.Generator().manual_seed(batch * 1000 + dim) + nodes, types = 97, 11 + edges, edge_types = _hub_graph(nodes, 20000, types, generator, 'cuda') + layout = csr_layout(edges, edge_types, nodes) + assert len(layout.hub_rows) and (layout.hub_first >= 0).sum() == 2 and 32 * HUB_TILES < 20000 // 6 + # Strided states and a batch-broadcast relation table exercise stride handling. + states = torch.randn(batch, nodes, 2 * dim, generator=generator)[..., ::2].to('cuda', dtype) + relations = torch.randn(1, types, dim, generator=generator).to('cuda', dtype).expand(batch, -1, -1) + boundary = torch.randn(batch, nodes, dim, generator=generator).to('cuda', dtype) + previous = torch.are_deterministic_algorithms_enabled() + try: + torch.use_deterministic_algorithms(True) + actual = distmult_sum(states, boundary, relations, layout) + finally: + torch.use_deterministic_algorithms(previous) + expected = _single_pass_rows()(states, boundary, relations, layout) + assert torch.equal(actual.view(torch.int16 if dtype != torch.float32 else torch.int32), + expected.view(torch.int16 if dtype != torch.float32 else torch.int32)) + + +def _sequential_ordered(): + """The previous one-edge-per-iteration ordered kernel, retained as the bitwise oracle.""" + + @triton.jit + def reduce(S, R, O, PTR, SRC, REL, + SN: tl.constexpr, SB: tl.constexpr, SD: tl.constexpr, + RN: tl.constexpr, RB: tl.constexpr, RD: tl.constexpr, + B: tl.constexpr, D: tl.constexpr, WIDTH: tl.constexpr, KIND: tl.constexpr): + row, batch = tl.program_id(0), tl.program_id(1) + feature = tl.arange(0, WIDTH) + start, end = tl.load(PTR + row), tl.load(PTR + row + 1) + value = tl.full((WIDTH,), 0. if KIND == 'sum' else -3.4028234663852886e38 if KIND == 'max' else 3.4028234663852886e38, tl.float32) + for edge in range(start, end): + source, relation = tl.load(SRC + edge), tl.load(REL + edge) + state = tl.load(S + source * SN + batch * SB + feature * SD, feature < D, other=0) + rel = tl.load(R + relation * RN + batch * RB + feature * RD, feature < D, other=0) + message = libdevice.mul_rn(state, rel) + if KIND == 'sum': + value = libdevice.add_rn(value, message) + elif KIND == 'max': + value = tl.maximum(value, message) + else: + value = tl.minimum(value, message) + tl.store(O + (row * B + batch) * D + feature, value, feature < D) + + def launch(states, relations, edges, kind): + nodes, batch, dim = states.shape + output = torch.empty((nodes, batch, dim), device=states.device) + reduce[(nodes, batch)](states, relations, output, *edges, *states.stride(), *relations.stride(), + batch, dim, triton.next_power_of_2(dim), kind, num_warps=4, enable_fp_fusion=False) + return output + return launch + + +@CUDA +@pytest.mark.parametrize('batch', [1, 6]) +@pytest.mark.parametrize('dim', [7, 32, 64]) +@pytest.mark.parametrize('warps', [1, 4]) +def test_scheduled_unrolled_ordered_reductions_are_bitwise_equal(batch, dim, warps): + pytest.importorskip('triton') + from dicee.query_answering.methods._ordered_aggregation import aggregate, layout + from dicee.query_answering.methods._ordered_aggregation_cuda import _schedule + generator = torch.Generator().manual_seed(batch * 100 + dim) + nodes, types = 61, 5 + edges, edge_types = _hub_graph(nodes, 3001, types, generator, 'cuda') + csr = layout(edges, edge_types, nodes, relation_order=dim == 32) + order, hubs = _schedule(csr[0]) + assert hubs == 2 and order[:2].tolist() == [1, nodes - 1] and sorted(order.tolist()) == list(range(nodes)) + # (N, B, D) views of batch-major storage, as used by UltraQuery, and broadcast relations. + states = torch.randn(batch, nodes, dim, generator=generator).cuda().transpose(0, 1) + states[3, 0, 0] = 1e30 # cancellation is order sensitive + relations = torch.randn(types, 1, dim, generator=generator).cuda().expand(-1, batch, -1) + oracle = _sequential_ordered() + for kind in ('sum', 'max', 'min'): + assert torch.equal(aggregate(states, relations, csr, kind, num_warps=warps), oracle(states, relations, csr, kind)) + fused = aggregate(states, relations, csr, 'pna', num_warps=warps) + separate = (oracle(states, relations, csr, 'sum'), oracle(states.square(), relations.square(), csr, 'sum'), + oracle(states, relations, csr, 'max'), oracle(states, relations, csr, 'min')) + assert all(torch.equal(a, b) for a, b in zip(fused, separate)) + cpu = aggregate(states.cpu(), relations.cpu(), tuple(x.cpu() for x in csr), 'pna') + for actual, expected in zip(fused, cpu): + torch.testing.assert_close(actual.cpu(), expected, rtol=0, atol=0) + + +@CUDA +def test_hub_split_handles_graphs_without_hubs_and_empty_batches(): + pytest.importorskip('triton') + from dicee.models._fused_message import csr_layout + from dicee.models._triton_message import distmult_sum + edges = torch.tensor([[0, 0, 1, 3], [1, 2, 0, 1]], device='cuda') + types = torch.tensor([0, 1, 0, 1], device='cuda') + layout = csr_layout(edges, types, 4) + assert not len(layout.hub_rows) and (layout.hub_first < 0).all() + states, boundary = torch.randn(2, 4, 5, device='cuda'), torch.randn(2, 4, 5, device='cuda') + relations = torch.randn(2, 2, 5, device='cuda') + previous = torch.are_deterministic_algorithms_enabled() + try: + torch.use_deterministic_algorithms(True) + actual = distmult_sum(states, boundary, relations, layout) + assert distmult_sum(states[:0], boundary[:0], relations[:0], layout).shape == (0, 4, 5) + finally: + torch.use_deterministic_algorithms(previous) + assert torch.equal(actual, _single_pass_rows()(states, boundary, relations, layout)) + + +@pytest.mark.parametrize('bad', [float('nan'), float('inf')]) +@CUDA +def test_deferred_device_checks_raise_and_drop_rejected_rows(bad): + from dicee.query_answering import QueryAnswerer + values = torch.randn(4, 2, 4, generator=torch.Generator().manual_seed(3), dtype=torch.float64) + model = TableModel(values).cuda().eval() + with torch.no_grad(): + model.table[1, 1, 2] = bad + engine = QueryAnswerer(model) + with pytest.raises(ValueError, match='finite complete'): + engine.predict((0, (0, 1))) + assert not engine._cache and not model.training + with torch.no_grad(): + model.table[1, 1, 2] = 0. + # A fresh engine shows that no rejected row survived in either cache. + expected = QueryAnswerer(TableModel(model.table.detach()).cuda().eval()).predict((0, (0, 1))) + assert torch.equal(engine.predict((0, (0, 1))), expected) + + +def test_scorers_share_model_context_until_the_graph_changes(monkeypatch): + from dicee.query_answering import QueryAnswerer, QueryContext + model = ULTRA(dict(num_entities=3, num_relations=2, ultra_dim=8, ultra_num_layers=2)) + model.set_graph([(0, 0, 1)], inverse_relations={0: 1}).eval() + calls, original = [], QueryContext.from_model + monkeypatch.setattr(QueryContext, 'from_model', classmethod(lambda cls, m: calls.append(m) or original(m))) + first, second = QueryAnswerer(model), QueryAnswerer(model) + assert len(calls) == 1 and first.scorer.context is second.scorer.context + model.set_graph([(0, 0, 2)], inverse_relations={0: 1}) + first.predict((0, (0,))) + second.predict((0, (0,))) + assert len(calls) == 2 and first.scorer.context is second.scorer.context + assert (0, 0, 2) in first.scorer.context.triples diff --git a/tests/test_kgfm_next_inference.py b/tests/test_kgfm_next_inference.py index a75a27eec..97340d9f2 100644 --- a/tests/test_kgfm_next_inference.py +++ b/tests/test_kgfm_next_inference.py @@ -13,15 +13,46 @@ FACTS = torch.tensor([[0, 0, 1], [0, 0, 2], [1, 1, 2], [2, 0, 3], [3, 1, 0]]) +def ieee_cudnn(): + """Turn off TF32 in cuDNN, as the KGFM benchmark runners do, and return a function that restores the settings. + + The reference fixtures are IEEE float32 CPU runs. cuDNN uses TF32 by default, also for Flock's GRU, and + ``torch.set_float32_matmul_precision`` does not change that: the Flock scores then differ by about 1e-3. + The precision objects come from the class: after ``torch.backends.cudnn.rnn`` is imported, the module + attribute of that name is the submodule (see ``dicee.models._inference.float32_precision_backends``). + The legacy ``allow_tf32`` is not read on this path, because PyTorch rejects a mix of the two APIs. + """ + cudnn = type(torch.backends.cudnn) + backends = [backend for backend in (getattr(cudnn, 'conv', None), getattr(cudnn, 'rnn', None)) + if hasattr(backend, 'fp32_precision')] + if backends: + previous = [backend.fp32_precision for backend in backends] + for backend in backends: + backend.fp32_precision = 'ieee' + + def restore(): + for backend, value in zip(backends, previous): + backend.fp32_precision = value + return restore + previous = torch.backends.cudnn.allow_tf32 + torch.backends.cudnn.allow_tf32 = False + + def restore(): + torch.backends.cudnn.allow_tf32 = previous + return restore + + @pytest.fixture(autouse=True) def small_thread_pool(): previous = torch.get_num_threads() previous_precision = torch.get_float32_matmul_precision() torch.set_float32_matmul_precision('highest') + restore_cudnn = ieee_cudnn() try: torch.set_num_threads(2) yield finally: + restore_cudnn() torch.set_num_threads(previous) torch.set_float32_matmul_precision(previous_precision) diff --git a/tests/test_kgicl.py b/tests/test_kgicl.py new file mode 100644 index 000000000..ee4d25d89 --- /dev/null +++ b/tests/test_kgicl.py @@ -0,0 +1,446 @@ +"""Official KG-ICL parity and DICE graph, prompt, scoring and training contracts.""" +import hashlib +import json +import os +from pathlib import Path + +import numpy as np +import pytest +import torch + +from dicee.config import Namespace +from dicee.models.kgicl import KGICL, UPSTREAM_COMMIT +from dicee.models.kgicl_prompts import PromptGraph, PromptSampler, answer_distance_rates + +FIXTURES = Path(__file__).parent / "fixtures" / "kgicl" +SMALL = dict(kgicl_dim=8, kgicl_attn_dim=3, kgicl_num_layers=3, kgicl_prompt_layers=2) + + +@pytest.fixture(autouse=True) +def small_thread_pool(): + previous = torch.get_num_threads() + previous_precision = torch.get_float32_matmul_precision() + torch.set_float32_matmul_precision('highest') + torch.set_num_threads(2) + yield + torch.set_num_threads(previous) + torch.set_float32_matmul_precision(previous_precision) + + +@pytest.fixture +def facts(): + return torch.tensor([[0, 0, 1], [0, 0, 2], [1, 1, 2], [2, 0, 3], [3, 1, 0], [3, 2, 4], [4, 0, 0], + [1, 0, 3], [2, 2, 2], [4, 1, 1], [0, 2, 3], [2, 0, 4]]) + + +def make_model(facts, **kwargs): + return KGICL(dict(num_entities=6, num_relations=3, **SMALL, **kwargs)).set_graph(facts) + + +def load_fixture(name): + fixture = torch.load(FIXTURES / f"{name}.pt", weights_only=False) + assert fixture["upstream_commit"] == UPSTREAM_COMMIT + assert fixture["patch_sha256"] == hashlib.sha256((FIXTURES / "upstream-fixes.patch").read_bytes()).hexdigest() + args = dict(num_entities=fixture["num_entities"], num_relations=fixture["num_relations"], kgicl_dim=fixture["dim"], + kgicl_attn_dim=fixture["attn_dim"], kgicl_num_layers=fixture["num_layers"], + kgicl_prompt_layers=fixture["prompt_layers"], kgicl_shots=fixture["shots"]) + model = KGICL(args) + if name == "tiny": + state = model.state_dict() + assert set(fixture["state_dict"]) == {k for k in state if ".conv." not in k} + model.load_state_dict({**state, **fixture["state_dict"]}, strict=True) + else: + directory = os.environ.get("KGICL_CHECKPOINT_DIR") + if directory is None: + pytest.skip("Set KGICL_CHECKPOINT_DIR to the directory with the official KG-ICL-6L model_best.tar") + path = Path(directory) / "model_best.tar" + assert hashlib.sha256(path.read_bytes()).hexdigest() == fixture["checkpoint_sha256"] + model.load_pretrained(path) + return fixture, model.set_graph(fixture["graph"]) + + +def replay(fixture): + return {q: [PromptGraph(g["edge_index"], g["edge_type"], g["labels"], g["head"], g["tail"]) for g in graphs] + for q, graphs in fixture["prompts"].items()} + + +def score_queries(model, queries, num_direct): + """All-entity scores for internal (head, relation) queries; inverse relations use head prediction.""" + rows = [] + for head, relation in queries.tolist(): + if relation < num_direct: + rows.append(model(torch.tensor([[head, relation]]))) + else: + rows.append(model.forward_k_vs_all_heads(torch.tensor([[relation - num_direct, head]]))) + return torch.cat(rows) + + +def test_official_architecture(): + model = KGICL({}) + assert len(model.state_dict()) == 220 + assert sum(p.numel() for p in model.parameters()) == 90856 + assert not hasattr(model, "entity_embeddings") + with pytest.raises(RuntimeError, match="Attach"): + model(torch.tensor([[0, 0, 1]])) + + +@pytest.mark.parametrize("name", ["tiny", "official"]) +def test_upstream_numerical_parity(name): + fixture, model = load_fixture(name) + model.eval().use_prompts(replay(fixture)) + tolerance = dict(atol=2e-5, rtol=1e-4) + with torch.no_grad(): + for relation, expected in fixture["prompt_outputs"].items(): + torch.testing.assert_close(model.encode_prompts(relation), expected, **tolerance) + scores = score_queries(model, fixture["queries"], fixture["num_relations"]) + torch.testing.assert_close(scores, fixture["unmasked_scores"], **tolerance) + rates = answer_distance_rates(fixture["valid"], fixture["train"], fixture["num_entities"]) + assert rates == fixture["answer_distance"] + model.masked_distances = tuple(d for d in range(fixture["num_layers"] + 1) if rates[d] < 0.02) + model.clear_inference_cache() + masked = score_queries(model, fixture["queries"], fixture["num_relations"]) + torch.testing.assert_close(masked, fixture["scores"], **tolerance) + assert (masked == 0).sum() > (scores == 0).sum() + model.masked_distances = () + model.clear_inference_cache() + # Evaluation-mode gradients; upstream's checkpoint keeps unused NBFNet parameters. + (score_queries(model, fixture["queries"], fixture["num_relations"]) * fixture["gradient_weights"]).sum().backward() + for parameter_name, parameter in model.named_parameters(): + expected = fixture["gradients"].get(parameter_name) + if expected is None: + assert parameter.grad is None, parameter_name + else: + torch.testing.assert_close(parameter.grad, expected, atol=1e-4, rtol=1e-4, + msg=lambda msg: parameter_name + ": " + msg) + + +def test_prompt_extraction_matches_upstream(): + fixture = torch.load(FIXTURES / "tiny.pt", weights_only=False) + sampler = PromptSampler(fixture["train"], fixture["num_entities"], fixture["num_relations"], + list(range(fixture["num_relations"]))) + checked = 0 + for relation, cases in fixture["cases"].items(): + for case in cases: + example = sampler.extract(*case["example"]) + entities = example.entities.tolist() + assert sorted(entities) == sorted(case["entities"]) + assert dict(zip(entities, example.labels.tolist())) == dict(zip(case["entities"], case["labels"])) + triples = {(entities[h], r, entities[t]) for h, r, t in example.triples.tolist()} + assert triples == set(case["triples"]) and len(triples) == len(example.triples) + assert example.self_loop == (case["example"][0] == case["example"][1]) + checked += 1 + assert checked > 40 and any(c["example"][0] == c["example"][1] for cases in fixture["cases"].values() for c in cases) + + +def test_prompt_sampling_is_seeded_and_order_independent(facts): + model = make_model(facts).eval() + other = make_model(facts).eval() + for relation in (0, 1, 2, 4, 5): + first = model.prompt_graphs(relation) + assert len(first) == 5 + again = other.prompt_graphs(relation) + for a, b in zip(first, again): + assert torch.equal(a.edge_index, b.edge_index) and torch.equal(a.labels, b.labels) + assert (a.head, a.tail) == (b.head, b.tail) + # Relation 0 has six facts: five distinct examples. Relation 1 has three: slots cycle. + assert len(model.sampler.examples(0)) == 5 and len(model.sampler.examples(1)) == 3 + slots = model.prompt_graphs(1) + assert slots[3] is not slots[0] and torch.equal(slots[3].edge_index, slots[0].edge_index) + # Inverse relations reuse the examples with swapped head and tail tokens. + assert [(g.head, g.tail) for g in model.prompt_graphs(4)] == [(1, 0) if not e.self_loop else (0, 0) + for e in [model.sampler.examples(1)[j % 3] for j in range(5)]] + seeded = make_model(facts, kgicl_prompt_seed=1).eval() + assert any(not torch.equal(a.entities, b.entities) for a, b in zip(model.sampler.examples(0), seeded.sampler.examples(0))) + queries = torch.tensor([[0, 0], [1, 1], [2, 2], [3, 0], [4, 1], [0, 2]]) + with torch.no_grad(): + expected = model(queries) + reversed_model = make_model(facts).eval() + reversed_model.load_state_dict(model.state_dict()) + actual = reversed_model(queries.flip(0)).flip(0) + reversed_model.query_batch_size = 1 + single = reversed_model(queries) + assert torch.equal(expected, actual) and torch.equal(expected, single) + + +def test_self_loop_example_puts_both_tokens_on_the_example_entity(): + model = KGICL(dict(num_entities=4, num_relations=1, **SMALL)).set_graph(torch.tensor([[0, 0, 0], [0, 0, 1], [1, 0, 2]])) + examples = model.sampler.examples(0) + loop = next(e for e in examples if e.self_loop) + assert loop.labels[0].tolist() == [0, 0] and loop.entities[0] == 0 + graphs = model.sampler.prompts(0, [next(i for i, e in enumerate(examples) if e is loop)]) + assert (graphs[0].head, graphs[0].tail) == (0, 0) + # Each self-loop fact appears once, with its inverse. + assert (graphs[0].edge_index[0] == graphs[0].edge_index[1]).sum() == 2 + + +def test_relation_without_facts_uses_upstream_fallback(facts): + model = KGICL(dict(num_entities=6, num_relations=4, **SMALL)).set_graph(facts).eval() + for relation, (head, tail) in ((3, (0, 1)), (7, (1, 0))): + graphs = model.prompt_graphs(relation) + assert len(graphs) == 5 and all(g is graphs[0] for g in graphs) + assert graphs[0].edge_index.tolist() == [[0], [1]] and graphs[0].edge_type.tolist() == [relation] + assert (graphs[0].head, graphs[0].tail) == (head, tail) + with torch.no_grad(): + assert torch.isfinite(model(torch.tensor([[0, 3]]))).all() + + +@pytest.mark.parametrize("seed", [0, 3]) +def test_scoring_interfaces_and_chunking(facts, seed): + model = make_model(facts, kgicl_prompt_seed=seed).eval() + queries, target = facts[:, :2], facts[:, 2:] + with torch.no_grad(): + all_scores = model(queries) + torch.testing.assert_close(model(facts), all_scores.gather(1, target).flatten()) + torch.testing.assert_close(model((queries, target)), all_scores.gather(1, target)) + torch.testing.assert_close(model(queries, torch.tensor([0, 1])), all_scores[:, :2]) + groups = facts[:, None].repeat(1, all_scores.shape[1], 1) + groups[..., 2] = torch.arange(all_scores.shape[1]) + torch.testing.assert_close(model(groups), all_scores) + torch.testing.assert_close(model(facts[[2, 1, 2, 0]]), model(facts)[[2, 1, 2, 0]]) + heads = model.forward_k_vs_all_heads(facts[:, 1:]) + assert torch.isfinite(all_scores).all() and torch.isfinite(heads).all() + assert model(torch.empty(0, 3, dtype=torch.long)).shape == (0,) + assert model(torch.empty(0, 2, dtype=torch.long)).shape == (0, all_scores.shape[1]) + # Entity 5 has no facts: it is never reached and scores exactly zero. + assert (all_scores[:, 5] == 0).all() + + +def test_reached_set_and_distance_mask(facts): + model = make_model(facts).eval() + with torch.no_grad(): + scores = model(torch.tensor([[0, 0]])) + _, depth = model._expand_scores(torch.tensor([0]), torch.tensor([0]), + model.encode_prompts(0)[None], (model.edge_index, model.edge_type)) + model.masked_distances = (0, 1) + model.clear_inference_cache() + masked = model(torch.tensor([[0, 0]])) + assert depth[0, 0] == 0 and depth[0, 5] == -1 and (depth[0, [1, 2, 3, 4]] == 1).all() + assert (masked[0, :5] == 0).all() and scores[0, :5].ne(0).all() + + +def test_graph_lifecycle_and_equivariance(facts, tmp_path): + model = make_model(facts).eval() + prompts = {q: model.prompt_graphs(q) for q in range(6)} + model.use_prompts(prompts) + with torch.no_grad(): + before = model(facts) + entities = torch.tensor([3, 5, 1, 0, 2, 4]) + permuted = torch.stack((entities[facts[:, 0]], facts[:, 1], entities[facts[:, 2]]), 1) + model.set_graph(permuted).use_prompts(prompts) + with torch.no_grad(): + torch.testing.assert_close(model(permuted), before, atol=1e-5, rtol=1e-4) + model.set_graph(facts) + assert not model._prompt_overrides + model.save_graph(tmp_path / "kgicl_graph.pt") + restored = make_model(facts).eval() + restored.load_state_dict(model.state_dict(), strict=True) + restored.load_graph(tmp_path / "kgicl_graph.pt") + with torch.no_grad(): + torch.testing.assert_close(restored(facts), model(facts)) + assert not any("edge_index" in key or "graph" in key or "relation_id_map" in key for key in model.state_dict()) + model.set_graph(torch.tensor([[7, 2, 6]]), num_entities=8, num_relations=3) + with torch.no_grad(): + assert torch.isfinite(model(torch.tensor([[7, 2, 5]]))).all() + with pytest.raises(ValueError, match="vocabulary"): + model.load_graph(tmp_path / "kgicl_graph.pt") + + +def test_inverse_mapping(facts): + model = make_model(facts).eval() + prompts = {q: model.prompt_graphs(q) for q in range(6)} + model.use_prompts(prompts) + with torch.no_grad(): + before, heads = model(facts), model.forward_k_vs_all_heads(facts[:, 1:]) + forward, inverse = facts.clone(), facts[:, [2, 1, 0]].clone() + forward[:, 1] *= 2 + inverse[:, 1] = inverse[:, 1] * 2 + 1 + model.set_graph(torch.cat((forward, inverse)), 6, 6, {0: 1, 2: 3, 4: 5}).use_prompts(prompts) + with torch.no_grad(): + torch.testing.assert_close(model(forward), before) + torch.testing.assert_close(model(inverse[:, :2]), heads) + # Prompt examples are seeded by the public direct relation ID. + assert model.sampler.public_relations == [0, 2, 4] + + +def test_training_masks_and_gradients(facts): + model = make_model(facts) + captured = [] + original = model._expand_scores + model._expand_scores = lambda *a: captured.append(a[3]) or original(*a) + scores = model(facts[:1, :2]) + index, types = captured[0] + kept = set(map(tuple, torch.stack((index[0], types, index[1]), 1).tolist())) + assert (0, 0, 1) not in kept and (1, 3, 0) not in kept and (0, 0, 2) not in kept + assert (1, 1, 2) in kept + scores.square().mean().backward() + assert all(torch.isfinite(p.grad).all() for p in model.parameters() if p.grad is not None) + assert model.relation_encoder.W_message[0].weight.grad.abs().sum() > 0 + assert model.gnn_layers[0].W_h.weight.grad.abs().sum() > 0 + + +def test_checkpoint_validation(facts, tmp_path): + model = make_model(facts) + state = model.state_dict() + path = tmp_path / "model_best.tar" + for wrapped in ("bare", "model", "state_dict"): + payload = state if wrapped == "bare" else {wrapped: state, "optimizer_state_dict": {}, "epoch_id": 3} + torch.save(payload, path) + model.load_pretrained(path) + for invalid in ({**state, "extra": torch.ones(1)}, dict(list(state.items())[1:]), KGICL({}).state_dict()): + torch.save({"state_dict": invalid}, path) + with pytest.raises(ValueError, match="architecture mismatch"): + model.load_pretrained(path) + + +def test_invalid_settings_and_ids(facts): + for args in ({"kgicl_dim": 0}, {"kgicl_shots": 0}, {"kgicl_query_batch_size": 0}, {"kgicl_prompt_seed": -1}, + {"kgicl_masked_distances": [-1]}, {"normalization": "LayerNorm"}, {"byte_pair_encoding": True}): + with pytest.raises(ValueError): + KGICL(args) + model = make_model(facts).eval() + for triples in (torch.tensor([[6, 0, 1]]), torch.tensor([[0, 3, 1]]), torch.tensor([[-1, 0, 1]])): + with pytest.raises(ValueError, match="vocabulary"): + model(triples) + with pytest.raises(ValueError): + model.use_prompts({0: []}) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is unavailable") +@pytest.mark.parametrize("deterministic", [False, True]) +def test_cuda_fused_path_matches_and_is_batch_invariant(facts, deterministic, monkeypatch): + pytest.importorskip("triton") + # PyTorch requires this for deterministic cuBLAS; the CQA Docker image sets it. + monkeypatch.setenv("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + model = KGICL(dict(num_entities=6, num_relations=3, kgicl_query_batch_size=4)).set_graph(facts).eval().requires_grad_(False) + queries = torch.cat((facts[:, :2], torch.tensor([[5, 0], [1, 2]]))) + with torch.no_grad(): + cpu, cpu_heads = model(queries), model.forward_k_vs_all_heads(queries[:, [1, 0]]) + model.cuda() + previous = torch.are_deterministic_algorithms_enabled() + try: + torch.use_deterministic_algorithms(deterministic) + for backend in ("torch", "triton"): + model.set_inference_backend(backend) + with torch.no_grad(): + batched = model(queries) + model.query_batch_size = 1 + single = model(queries) + reordered = model(queries.flip(0)).flip(0) + model.query_batch_size = 4 + heads = model.forward_k_vs_all_heads(queries[:, [1, 0]].cuda()) + torch.testing.assert_close(batched.cpu(), cpu, atol=2e-4, rtol=2e-4) + torch.testing.assert_close(heads.cpu(), cpu_heads, atol=2e-4, rtol=2e-4) + # Triton rows never use atomics; PyTorch's CUDA scatter needs deterministic algorithms. + if backend == "triton" or deterministic: + assert torch.equal(batched, single) and torch.equal(batched, reordered) + finally: + torch.use_deterministic_algorithms(previous) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is unavailable") +def test_cuda_fused_hub_rows_and_query_blocks(): + """Hub rows are summed in fixed segments and query blocks are padded; a row never depends on its batch.""" + pytest.importorskip("triton") + from dicee.models._triton_kgicl import HUB_SEGMENTS, QUERIES, SEGMENT + generator = torch.Generator().manual_seed(0) + n = 3 * SEGMENT * HUB_SEGMENTS + + def relations(count): + return torch.randint(0, 3, (count,), generator=generator) + # Entity 0 receives an edge from every other entity, a hub row of several segments. + spokes = torch.stack((torch.arange(1, n), relations(n - 1), torch.zeros(n - 1, dtype=torch.long)), 1) + noise = torch.stack((torch.randint(0, n, (4 * n,), generator=generator), relations(4 * n), + torch.randint(0, n, (4 * n,), generator=generator)), 1) + model = KGICL(dict(num_entities=n, num_relations=3, kgicl_masked_distances=[0, 2], kgicl_query_batch_size=2 * QUERIES + 1)) + model = model.set_graph(torch.cat((spokes, noise))).eval().requires_grad_(False).cuda() + queries = torch.cat((torch.tensor([[0, 0], [0, 1], [5, 2]]), noise[:10, :2])).cuda() + with torch.no_grad(): + model.set_inference_backend("torch") + expected = model(queries) + model.set_inference_backend("triton") + batched = model(queries) + model.query_batch_size = 1 + single = model(queries) + model.query_batch_size = 3 + reordered = model(queries.flip(0)).flip(0) + torch.testing.assert_close(batched, expected, atol=2e-4, rtol=2e-4) + assert torch.equal(batched, single) and torch.equal(batched, reordered) + # Heads and entities first reached after two hops are masked. + assert (batched[torch.arange(len(queries)), queries[:, 0]] == 0).all() and (batched != 0).any() + + +@pytest.mark.parametrize("model_name,technique,grouped,trainer", [ + ("KGICL", "NegSample", False, "torchCPUTrainer"), + ("KGICL", "NegSample", True, "torchCPUTrainer"), + ("KGICL", "KvsAll", False, "torchCPUTrainer"), + ("KGICL", "1vsAll", False, "PL"), +]) +def test_execute_and_reload(tmp_path, model_name, technique, grouped, trainer): + from dicee.executer import Execute + from dicee.knowledge_graph_embeddings import KGE + dataset = tmp_path / "data" + dataset.mkdir() + (dataset / "train.txt").write_text("a r b\na r c\nb s c\nc r d\nd s a\nb r d\n") + (dataset / "valid.txt").write_text("a s d\n") + (dataset / "test.txt").write_text("d r b\n") + args = Namespace() + args.model = model_name + args.kgicl_dim, args.kgicl_attn_dim, args.kgicl_num_layers, args.kgicl_prompt_layers = 8, 3, 3, 2 + args.dataset_dir, args.path_to_store_single_run = str(dataset), str(tmp_path / "run") + args.num_epochs, args.batch_size, args.neg_ratio, args.lr = 1, 2, 2, 0.001 + args.scoring_technique, args.trainer = technique, trainer + args.backend, args.separator = "pandas", " " + args.strict_negative_sampling = grouped + args.adversarial_temperature = 1.0 if grouped else None + if trainer == "PL": + args.pl_trainer_kwargs = {"accelerator": "cpu", "devices": 1} + execute = Execute(args) + report = execute.start() + assert np.isfinite(report["Test"]["MRR"]) + restored = KGE(path=str(tmp_path / "run")) + triples = torch.tensor([[0, 0, 1]]) + with torch.no_grad(): + torch.testing.assert_close(restored.model(triples), execute.trained_model.eval()(triples)) + assert (tmp_path / "run" / "kgicl_graph.pt").is_file() + restored.predict_missing_head_entity("r", "b") + restored.predict_missing_tail_entity("a", "r") + + +def test_zero_epoch_official_checkpoint_format(tmp_path): + from dicee.executer import Execute + dataset = tmp_path / "data" + dataset.mkdir() + (dataset / "train.txt").write_text("a r b\nb s c\nc r a\na s c\n") + (dataset / "test.txt").write_text("a s b\n") + path = tmp_path / "model_best.tar" + pretrained = KGICL({}) + torch.save({"state_dict": pretrained.state_dict(), "optimizer_state_dict": {}, "epoch_id": 22}, path) + args = Namespace() + args.model, args.kgicl_checkpoint = "KGICL", str(path) + args.dataset_dir, args.path_to_store_single_run = str(dataset), str(tmp_path / "run") + args.num_epochs, args.batch_size = 0, 2 + args.scoring_technique, args.eval_model = "KvsAll", "test" + report = Execute(args).start() + state = torch.load(tmp_path / "run" / "model.pt", weights_only=True) + for key in state: + torch.testing.assert_close(state[key], pretrained.state_dict()[key]) + assert np.isfinite(report["Test"]["MRR"]) + assert json.loads((tmp_path / "run" / "configuration.json").read_text())["kgicl_dim"] == 32 + + +def test_query_rows_are_bitwise_independent_of_batching(facts, monkeypatch): + """The query evaluator caches rows across beams and controls; batching must not change a row.""" + from dicee.query_answering.context import QueryContext, attached_context + from dicee.query_answering.engine import AtomicScorer + from dicee.query_answering.method_evaluation import deterministic_kgfm + monkeypatch.setenv("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + context = QueryContext([tuple(t) for t in facts.tolist()], 6, 3) + conditions = [(h, r) for h in range(6) for r in range(3)] + for device in ["cpu"] + (["cuda"] if torch.cuda.is_available() else []): + model = KGICL(dict(num_entities=1, num_relations=1, kgicl_query_batch_size=4)).eval().requires_grad_(False).to(device) + with deterministic_kgfm(), attached_context(model, context): + reference = AtomicScorer(model, context, row_batch_size=1).rows(conditions) + for size in (2, 5, 18): + rows = AtomicScorer(model, context, row_batch_size=size).rows(conditions[::-1]).flip(0) + assert torch.equal(rows, reference), (device, size) + assert torch.isfinite(reference).all() and (reference != 0).any() diff --git a/tests/test_neg_sample_margin_model_coverage.py b/tests/test_neg_sample_margin_model_coverage.py index c35c9e39e..9e1fcd88d 100644 --- a/tests/test_neg_sample_margin_model_coverage.py +++ b/tests/test_neg_sample_margin_model_coverage.py @@ -8,7 +8,7 @@ matrix rather than gathering per-row scores, so NegSample-family techniques do not produce a correctly shaped batch for it. - Pyke is excluded per project convention (not covered by this sweep). -- ULTRA, TRIX, TRIXRelation, Flock, and FlockRelation only support the +- ULTRA, TRIX, TRIXRelation, Flock, FlockRelation, and KGICL only support the scoring techniques validated in static_preprocess_funcs.py (KvsAll and, for the entity-prediction variants, NegSample/FixedNegSample/1vsAll/1vsSample/ KvsSample). Their forward_triples loops over unique (head, relation) @@ -22,7 +22,7 @@ from dicee.executer import Execute from dicee.static_funcs import MODEL_REGISTRY -EXCLUDED_MODELS = {"Shallom", "Pyke", "ULTRA", "TRIX", "TRIXRelation", "Flock", "FlockRelation"} +EXCLUDED_MODELS = {"Shallom", "Pyke", "ULTRA", "TRIX", "TRIXRelation", "Flock", "FlockRelation", "KGICL"} NATIVE_MODELS = sorted(set(MODEL_REGISTRY) - EXCLUDED_MODELS - {"BytE"}) diff --git a/tests/test_query_baselines.py b/tests/test_query_baselines.py new file mode 100644 index 000000000..dd4c95c11 --- /dev/null +++ b/tests/test_query_baselines.py @@ -0,0 +1,557 @@ +"""Native baselines against independently generated, pinned upstream outputs.""" + +import itertools +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +from dicee.query_answering import BenchmarkQuery, QueryBenchmark, QueryContext, evaluate_benchmark +from dicee.query_answering._query import compile_query +from dicee.query_answering.benchmark import QueryMetrics +from dicee.query_answering.methods import REFERENCES, load_method +from dicee.query_answering.methods._common import union_branches + +ROOT = Path(__file__).parent / 'fixtures' / 'query_baselines' + + +def cases(): + result = [] + for method in ('cone', 'cqd', 'qto', 'clmpt', 'ultra', 'gnnqe'): + path = ROOT / f'{method}.pt' + if not path.exists(): + continue + fixture = torch.load(path, weights_only=True, map_location='cpu') + for index, case in enumerate(fixture['cases']): + result.append(pytest.param(fixture, case, id=f'{method}-{index}')) + return result + + +@pytest.fixture(autouse=True) +def cpu_threads(): + previous = torch.get_num_threads() + torch.set_num_threads(2) + yield + torch.set_num_threads(previous) + + +@pytest.mark.parametrize('fixture,case', cases()) +def test_checkpoint_scores_ranks_ties_and_metrics(tmp_path, fixture, case): + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'checkpoint.pt' + torch.save({'model_state_dict': case['state']}, path) + model = load_method(case['name'], path, context, **case['config']) + assert fixture['upstream_commit'] == REFERENCES[case['name']][1] + assert not model.training + assert not any(parameter.requires_grad for parameter in model.parameters()) + for shape, expected in case['scores'].items(): + actual = model.predict(fixture['queries'][shape]) + torch.testing.assert_close(actual, expected, atol=3e-5, rtol=3e-5, msg=shape) + if case['name'] == 'cone' and shape in ('2u', 'up'): + # The reference embeds all DNF branches in one projection batch. Batch composition changes the + # float bits, so rebuild that joint batch on this CPU instead of comparing bits recorded on another. + branches = union_branches(compile_query(fixture['queries'][shape])) + joint = [values.reshape(1, len(branches), -1) for values in model.embed_batch(branches)] + torch.testing.assert_close(actual, model.score_embeddings(joint)[0], atol=0, rtol=0, + msg='ConE union branches must use the reference joint projection batch') + assert torch.equal(actual.argsort(descending=True), expected.argsort(descending=True)), shape + assert torch.equal(actual[:, None] == actual[None], expected[:, None] == expected[None]), shape + for policy in ('sort', 'expected'): + evaluator = QueryMetrics(context.num_entities, candidates=range(context.num_entities - 1), tie_policy=policy) + assert evaluator(actual, {1}, {0, 3}) == evaluator(expected, {1}, {0, 3}), (shape, policy) + if 'adjacency' in case: + with torch.no_grad(): + for relation, expected in enumerate(case['adjacency']): + torch.testing.assert_close(model.rows(torch.arange(context.num_entities), relation), expected) + for batch in case.get('batches', {}).values(): + actual = model.predict_batch(batch['queries']) + torch.testing.assert_close(actual, batch['scores'], atol=3e-5, rtol=3e-5) + assert torch.equal(actual.argsort(-1), batch['scores'].argsort(-1)) + if not model.pre_norm: + assert not torch.allclose(actual[0], model.predict(batch['queries'][0])) + + +def test_all_reference_methods_present(): + for method in ('cone', 'cqd', 'qto', 'clmpt', 'ultra', 'gnnqe'): + assert (ROOT / f'{method}.pt').exists(), f'Missing upstream fixture for {method}' + + +def test_expected_ties_match_all_permutations(): + scores = torch.tensor([1., 1., .4, .4, .4, 0.]) + evaluator = QueryMetrics(len(scores), candidates=[0, 1, 2, 3, 4], tie_policy='expected') + result = evaluator(scores, {1}, {2, 4}) + # After filtering: one higher negative, one tied negative, and the target. + ranks = [2, 3] + assert result['mrr'] == pytest.approx(sum(1 / rank for rank in ranks) / 2) + assert result['hits1'] == 0 + assert result['hits3'] == 1 + scores = torch.ones(4) + values = [] + for order in itertools.permutations(range(4)): + values.append(1 / (order.index(0) + 1)) + result = QueryMetrics(4, tie_policy='expected')(scores, set(), {0}) + assert result['mrr'] == pytest.approx(sum(values) / len(values)) + assert result['hits1'] == .25 + assert result['hits3'] == .75 + assert result['mrr'] != QueryMetrics(4, tie_policy='average')(scores, set(), {0})['mrr'] + + +@pytest.mark.parametrize('policy', ['expected', 'average', 'optimistic', 'pessimistic']) +def test_tie_bounds_match_filtered_candidate_enumeration(policy): + generator = torch.Generator().manual_seed(42) + for _ in range(30): + scores = torch.randint(0, 5, (20,), generator=generator).float() + scores[0] = -torch.inf + candidates = set(torch.randperm(20, generator=generator)[:15].tolist()) + easy, hard = set(sorted(candidates)[:3]), set(sorted(candidates)[3:8]) + expected = [] + for target in hard: + negatives = candidates - easy - hard + start = 1 + sum(bool(scores[node] > scores[target]) for node in negatives) + ties = sum(bool(scores[node] == scores[target]) for node in negatives) + ranks = (range(start, start + ties + 1) if policy == 'expected' else + [start + {'average': .5, 'optimistic': 0., 'pessimistic': 1.}[policy] * ties]) + expected.append([sum(1 / r for r in ranks) / len(ranks), + *(sum(r <= k for r in ranks) / len(ranks) for k in (1, 3, 10))]) + result = QueryMetrics(20, candidates=candidates, tie_policy=policy)(scores, easy, hard) + assert list(result.values()) == pytest.approx(torch.tensor(expected, dtype=torch.float64).mean(0).tolist()) + + +def test_strict_checkpoint_and_vocabulary_validation(tmp_path): + fixture = torch.load(ROOT / 'qto.pt', weights_only=True) + case = fixture['cases'][0] + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'weights.pt' + torch.save(case['state'] | {'unexpected': torch.tensor(0.)}, path) + with pytest.raises(ValueError, match='keys'): + load_method('qto', path, context, **case['config']) + torch.save(case['state'], path) + wrong = QueryContext([], context.num_entities + 1, context.num_relations) + with pytest.raises(ValueError, match='vocabulary'): + load_method('qto', path, wrong, **case['config']) + + +@pytest.mark.parametrize('reference_batching', [False, True]) +def test_qto_cache_invalidation_and_chunking(tmp_path, reference_batching): + fixture = torch.load(ROOT / 'qto.pt', weights_only=True) + case = fixture['cases'][0] + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'weights.pt' + torch.save(case['state'], path) + model = load_method('qto', path, context, row_batch_size=2, cache_bytes=160, + reference_batching=reference_batching, **case['config']) + query = fixture['queries']['3p'] + torch.testing.assert_close(model.predict(query), case['scores']['3p']) + assert model.cache.bytes <= 160 + with torch.no_grad(): + model.model.entity_embeddings.weight.mul_(2) + cached = model.predict(query) + model.cache.capacity = 0 + model.cache.values.clear() + model.cache.bytes = 0 + torch.testing.assert_close(cached, model.predict(query), rtol=0, atol=0) + + +def test_cqd_negation_is_not_silently_reinterpreted(tmp_path): + fixture = torch.load(ROOT / 'cqd.pt', weights_only=True) + case = fixture['cases'][0] + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'weights.pt' + torch.save(case['state'], path) + model = load_method('cqd', path, context, **case['config']) + with pytest.raises(ValueError, match='negated'): + model.predict(fixture['queries']['2in']) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA allocation guard') +def test_cqd_memory_guard_precedes_dense_scoring(tmp_path, monkeypatch): + from dicee.query_answering.methods import cqd + fixture = torch.load(ROOT / 'cqd.pt', weights_only=True) + case = fixture['cases'][0] + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'weights.pt' + torch.save(case['state'], path) + model = load_method('cqd', path, context, device='cuda', **case['config']) + monkeypatch.setattr(torch.cuda, 'mem_get_info', lambda device: (0, 2**30)) + monkeypatch.setattr(torch.cuda, 'memory_reserved', lambda device: 0) + monkeypatch.setattr(torch.cuda, 'memory_allocated', lambda device: 0) + monkeypatch.setattr(cqd, 'complex_rows', lambda *args: pytest.fail('Attempted dense allocation')) + with pytest.raises(MemoryError, match='reference stage.*scratch'): + model.predict(fixture['queries']['1p']) + + +@pytest.mark.parametrize('chunk', [1, 100]) +def test_opt_in_cqd_negation_matches_cqda(tmp_path, chunk): + fixture = torch.load(ROOT / 'cqd-negation.pt', weights_only=True) + assert fixture['upstream_commit'] == '642ce042708be6247087c09c780b9deb47e941d3' + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'weights.pt' + for case in fixture['cases']: + torch.save(case['state'], path) + model = load_method(case['name'], path, context, row_batch_size=chunk, **case['config']) + for shape, expected in case['scores'].items(): + actual = model.predict(fixture['queries'][shape]) + torch.testing.assert_close(actual, expected, atol=3e-6, rtol=3e-6, msg=shape) + assert torch.equal(actual.argsort(), expected.argsort()), shape + assert torch.equal(actual[:, None] == actual[None], expected[:, None] == expected[None]), shape + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA reference attention') +def test_bounded_attention_preserves_full_transformer_sequence(): + from dicee.query_answering.methods._bounded_attention import BoundedAttention + torch.manual_seed(5) + model = torch.nn.TransformerEncoderLayer(2000, 8, 256, .1).cuda().eval() + inputs = torch.randn(1100, 2, 2000, device='cuda') + with torch.no_grad(): + expected = model(inputs) + with BoundedAttention(): + actual = model(inputs) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA ordered reduction') +@pytest.mark.parametrize('reduction', ['sum', 'max', 'min']) +def test_ordered_sparse_kernel_preserves_sequential_rounding(reduction): + from dicee.query_answering.methods._ordered_aggregation import aggregate, layout + torch.manual_seed(7) + edges = torch.tensor([[0, 0, 0, 2, 2], [1, 2, 3, 1, 4]]) + types = torch.tensor([2, 1, 0, 0, 2]) + states = torch.randn(5, 3, 17) + relations = torch.randn(4, 3, 17) + states[:3, 0, 0] = torch.tensor([1e20, -1e20, 1.]) + indices = layout(edges, types, len(states)) + expected = aggregate(states, relations, indices, reduction) + actual = aggregate(states.cuda(), relations.cuda(), tuple(x.cuda() for x in indices), reduction) + torch.testing.assert_close(actual.cpu(), expected, atol=0, rtol=0) + + +def test_qto_empty_intermediate_stays_empty(tmp_path): + fixture = torch.load(ROOT / 'qto.pt', weights_only=True) + case = fixture['cases'][0] + context = QueryContext([], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'weights.pt' + torch.save(case['state'], path) + model = load_method('qto', path, context, threshold=1., negation_scale=1.) + assert torch.equal(model.predict(fixture['queries']['2p']), torch.zeros(context.num_entities)) + assert torch.equal(model.projection(torch.zeros(context.num_entities), 0, negate=True), torch.zeros(context.num_entities)) + + +def _shift_anchors(query, offset, n): + if isinstance(query, tuple) and len(query) == 2 and isinstance(query[0], int) and isinstance(query[1], tuple): + return (query[0] + offset) % n, query[1] + return tuple(_shift_anchors(part, offset, n) if isinstance(part, tuple) else part for part in query) + + +def _random_qto_pair(n=257, relations=5, seed=0, **options): + """Row-path and matrix-path QTO on one random graph and ComplEx state spanning several canonical blocks.""" + from dicee.query_answering.methods import qto + from dicee.query_answering.methods.checkpoints import _complex + generator = torch.Generator().manual_seed(seed) + triples = torch.stack([torch.randint(n, (4 * n,), generator=generator), torch.randint(relations, (4 * n,), generator=generator), + torch.randint(n, (4 * n,), generator=generator)], 1).tolist() + context = QueryContext(triples, n, relations) + # Small weights spread the softmax, so rows mix entries below and above the threshold. + state = {'embeddings.0.weight': .6 * torch.randn(n, 16, generator=generator), + 'embeddings.1.weight': .6 * torch.randn(relations, 16, generator=generator)} + config = dict(threshold=2e-4, negation_scale=3., reference_batching=True) + rows = qto.QTO(_complex(state, context), context, **config) + matrices = qto.QTO(_complex(state, context), context, **config | options) + return rows, matrices + + +@pytest.mark.parametrize('matrix_bytes,dense_elements', [(2**30, 2**24), (1, 2**24), (2**30, 300)]) +def test_qto_relation_matrices_reproduce_row_scores(monkeypatch, matrix_bytes, dense_elements): + from dicee.query_answering.methods import qto + fixture = torch.load(ROOT / 'qto.pt', weights_only=True) + monkeypatch.setattr(qto, 'DENSE_ELEMENTS', dense_elements) + rows, matrices = _random_qto_pair(matrix_bytes=matrix_bytes) + n = rows.context.num_entities + for shape, query in fixture['queries'].items(): + support = 0 + for offset in (0, 100, 230): + shifted = _shift_anchors(query, offset, n) + expected = rows.predict(shifted) + support = max(support, int(expected.count_nonzero())) + torch.testing.assert_close(matrices.predict(shifted), expected, atol=0, rtol=0, msg=shape) + assert support > 10, shape # the comparison covers nontrivial score vectors of every shape + if matrix_bytes > 1: + assert 0 < matrices.matrices.bytes['device'] <= matrix_bytes + else: + assert not matrices.matrices.tiers['device'] + + +def test_qto_few_heads_compute_rows_unless_the_matrix_is_resident(): + from unittest.mock import patch + + from dicee.query_answering.methods import qto + rows, matrices = _random_qto_pair(matrix_bytes=2**30) + n = rows.context.num_entities + anchor = torch.nn.functional.one_hot(torch.tensor(130), n).float() + dense = torch.rand(n, generator=torch.Generator().manual_seed(1)) + with torch.no_grad(): + torch.testing.assert_close(matrices.projection(anchor, 1), rows.projection(anchor, 1), atol=0, rtol=0) + assert not matrices.matrices.tiers['device'] + torch.testing.assert_close(matrices.projection(dense, 1), rows.projection(dense, 1), atol=0, rtol=0) + assert set(matrices.matrices.tiers['device']) == {1} + expected = rows.projection(anchor, 1, negate=True) + with patch.object(qto, 'complex_rows', wraps=qto.complex_rows) as scorer: + actual = matrices.projection(anchor, 1, negate=True) + assert scorer.call_count == 0 + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + +def test_qto_relation_matrices_match_upstream_fixture_and_invalidate(tmp_path): + fixture = torch.load(ROOT / 'qto.pt', weights_only=True) + case = fixture['cases'][0] + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'weights.pt' + torch.save(case['state'], path) + model = load_method('qto', path, context, reference_batching=True, matrix_bytes=2**20, **case['config']) + for shape, query in fixture['queries'].items(): + torch.testing.assert_close(model.predict(query), case['scores'][shape], msg=shape) + rows = load_method('qto', path, context, reference_batching=True, **case['config']) + with torch.no_grad(): + for method in (model, rows): + method.model.entity_embeddings.weight.mul_(2) + for shape, query in fixture['queries'].items(): + torch.testing.assert_close(model.predict(query), rows.predict(query), atol=0, rtol=0, msg=shape) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason='host spill of device matrices') +def test_qto_relation_matrices_spill_to_host_memory(): + fixture = torch.load(ROOT / 'qto.pt', weights_only=True) + rows, matrices = _random_qto_pair(matrix_bytes=40_000, host_matrix_bytes=2**30) + rows, matrices = rows.cuda(), matrices.cuda() + for shape, query in fixture['queries'].items(): + for offset in (0, 100, 230): + shifted = _shift_anchors(query, offset, rows.context.num_entities) + torch.testing.assert_close(matrices.predict(shifted), rows.predict(shifted), atol=0, rtol=0, msg=shape) + tiers = matrices.matrices.tiers + assert tiers['host'] and all(tensor.device.type == 'cpu' for matrix in tiers['host'].values() for tensor in matrix) + assert matrices.matrices.bytes['device'] <= 40_000 + + +def test_qto_relation_matrix_options_are_validated(): + with pytest.raises(ValueError, match='reference_batching'): + _random_qto_pair(n=7, matrix_bytes=2**20, reference_batching=False) + with pytest.raises(ValueError, match='reference_batching'): + _random_qto_pair(n=7, host_matrix_bytes=2**20) + with pytest.raises(ValueError, match='nonnegative'): + _random_qto_pair(n=7, matrix_bytes=2**20, host_matrix_bytes=-1) + + +def test_dual_tie_protocol_resume_uses_one_prediction(tmp_path): + data = QueryBenchmark('toy', 'transductive', 'test', QueryContext([], 5, 2), + tuple(BenchmarkQuery('1p', (head, (0,)), frozenset({1}), frozenset({3})) for head in range(3)), + tuple(range(5))) + scores = torch.tensor([.5, .5, .4, .5, .1]) + calls = [] + + def predict(query): + calls.append(query) + return scores + + def interrupt(done, total): + if done == 2: + raise InterruptedError + + options = dict(checkpoint_dir=tmp_path, checkpoint_identity={'model': 'toy'}, checkpoint_every=1, + additional_tie_policies=('expected',)) + with pytest.raises(InterruptedError): + evaluate_benchmark(data, predict, progress=interrupt, **options) + report = evaluate_benchmark(data, predict, **options) + assert len(calls) == len(set(calls)) == 3 + for policy, values in [('sort', report), ('expected', report['additional_tie_metrics']['expected'])]: + assert values['per_shape']['1p']['mrr'] == QueryMetrics(5, tie_policy=policy)(scores, {1}, {3})['mrr'] + + +def test_baseline_runner_end_to_end_and_resume(tmp_path, monkeypatch): + from dicee.query_answering.methods.cone import ConE + from dicee.scripts.evaluate_query_methods import main + from tests.test_query_benchmark import fixture_dataset + fixture_dataset(tmp_path / 'data', 'FB15k237LogicalQuery') + fixture = torch.load(ROOT / 'cone.pt', weights_only=True) + state = fixture['cases'][0]['state'] + state['entity_embedding'] = torch.cat((state['entity_embedding'], state['entity_embedding'][:1])) + path = tmp_path / 'weights.pt' + torch.save(state, path) + entry = dict(method='cone', checkpoint=str(path), dataset='FB15k237LogicalQuery', root=str(tmp_path / 'data'), + vocabulary_dataset='FB15k237LogicalQuery', options={'center_reg': .02}) + manifest = tmp_path / 'manifest.json' + manifest.write_text(json.dumps(entry)) + args = ['--manifest', str(manifest), '--output', str(tmp_path / 'results'), '--max-queries-per-shape', '1'] + main(args) + path = tmp_path / 'results' / 'cone-FB15k237LogicalQuery' / 'result.json' + report = json.loads(path.read_text()) + assert report['queries'] == 14 + assert report['additional_tie_metrics']['expected']['averages']['all']['shapes'] == 14 + monkeypatch.setattr(ConE, 'predict', lambda *args: pytest.fail('Resumed evaluation recomputed a query')) + main(args) + assert json.loads(path.read_text())['per_shape'] == report['per_shape'] + + +@pytest.mark.parametrize('chunk,reference_batching,final_batch', [(1, False, None), (3, False, None), (100, False, None), + (3, True, None), (3, True, 1)]) +def test_cqd_chunking_preserves_stage_normalization(tmp_path, chunk, reference_batching, final_batch): + fixture = torch.load(ROOT / 'cqd.pt', weights_only=True) + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'weights.pt' + for case in fixture['cases']: + torch.save(case['state'], path) + model = load_method(case['name'], path, context, row_batch_size=chunk, cache_bytes=160, + reference_batching=reference_batching, final_batch_size=final_batch, **case['config']) + for shape, expected in case['scores'].items(): + actual = model.predict(fixture['queries'][shape]) + torch.testing.assert_close(actual, expected, atol=2e-6, rtol=2e-6) + assert torch.equal(actual.argsort(), expected.argsort()) + assert model.cache.bytes <= 160 + + +def test_checkpoint_graphs_validate_ids_without_using_held_out_edges(): + from dicee.query_answering.methods.checkpoints import _validate_graph_buffer + context = QueryContext([(0, 0, 1), (1, 1, 2)], 3, 2) + graph = SimpleNamespace(_edge_list=torch.tensor([[0, 1, 0]]), _edge_weight=torch.ones(1), num_node=3, num_relation=2) + metadata = {} + _validate_graph_buffer('fact_graph', graph, context, metadata) + assert not metadata['fact_graph']['used_for_inference'] + graph._edge_list = torch.tensor([[0, 2, 0]]) + with pytest.raises(ValueError, match='training facts'): + _validate_graph_buffer('fact_graph', graph, context, metadata) + _validate_graph_buffer('graph', graph, context, metadata) + assert not metadata['graph']['used_for_inference'] + assert context.triples == ((0, 0, 1), (1, 1, 2)) + graph._edge_list = torch.tensor([[0, 2, 2]]) + with pytest.raises(ValueError, match='out-of-vocabulary'): + _validate_graph_buffer('graph', graph, context, metadata) + + +@pytest.mark.parametrize('index', range(3)) +def test_ultraquery_observed_traversal_is_an_exact_max_with_symbolic_traversal(tmp_path, index): + """The opt-in observed-fact traversal equals max(GNN projection, naive symbolic traversal) at every projection.""" + from dicee.query_answering.methods._common import fuzzy_combine + fixture = torch.load(ROOT / 'ultra.pt', weights_only=True, map_location='cpu') + case = fixture['cases'][index] + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'checkpoint.pt' + torch.save({'model_state_dict': case['state']}, path) + plain = load_method('ultraquery', path, context, **case['config']) + model = load_method('ultraquery', path, context, **case['config'], observed_traversal=True) + assert model.provenance['configuration']['observed_traversal'] is True + + def symbolic(source, relation): + result = torch.zeros_like(source) + for h, r, t in context.triples: + if r == relation: + result[t] = torch.maximum(result[t], source[h]) + return result + + def membership(tree): + if tree[0] == 'anchor': + return torch.nn.functional.one_hot(torch.tensor(tree[1]), context.num_entities).float() + if tree[0] == 'project': + source = membership(tree[2]) + return torch.maximum(plain.project(source, tree[1]), symbolic(source, tree[1])) + if tree[0] == 'not': + return 1 - membership(tree[1]) + return fuzzy_combine([membership(child) for child in tree[1:]], tree[0], plain.logic) + + with torch.no_grad(): + for shape, query in fixture['queries'].items(): + tree = compile_query(query) + torch.testing.assert_close(model.membership(tree), membership(tree), atol=0, rtol=0, msg=shape) + # Off by default: the plain port keeps the pinned upstream scores. + torch.testing.assert_close(plain.predict(query), case['scores'][shape], atol=3e-5, rtol=3e-5, msg=shape) + anchor, (relation,) = fixture['queries']['1p'] + tails = sorted(context.outgoing.get(anchor, {}).get(relation, ())) + assert tails and bool((model.membership(compile_query(fixture['queries']['1p']))[tails] == 1).all()) + batch = [fixture['queries'][shape] for shape in ('2p', '2p')] + torch.testing.assert_close(model.predict_batch(batch)[0], model.predict(batch[0]), atol=3e-5, rtol=3e-5) + + +@pytest.mark.parametrize('conditioning', ['direct', 'query']) +def test_frozen_ultra_with_query_conditioning_equals_the_ultraquery_projection(tmp_path, conditioning): + """UltraQuery's weights as a frozen ULTRA: with relation_conditioning='query', every atom equals UltraQuery's 1p projection; + ULTRA's link-prediction default conditions inverse queries on their direct relation instead.""" + from dicee.models import ULTRA + from dicee.query_answering.context import attached_context, evaluation_mode + fixture = torch.load(ROOT / 'ultra.pt', weights_only=True, map_location='cpu') + case = fixture['cases'][0] + # Declared inverse pairs make the context graph identical for both models. + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations'], ((0, 2), (1, 3))) + path = tmp_path / 'checkpoint.pt' + torch.save({'model_state_dict': case['state']}, path) + port = load_method('ultraquery', path, context, threshold=0.) + state = {key.removeprefix('model.model.').removeprefix('model.'): value for key, value in case['state'].items()} + torch.save({'model': state}, tmp_path / 'ultra.pth') + dim = state['relation_model.layers.0.linear.weight'].shape[0] + layers = len({key.split('.')[2] for key in state if key.startswith('relation_model.layers.')}) + model = ULTRA(dict(num_entities=1, num_relations=1, graph_inference_backend='torch', ultra_dim=dim, ultra_num_layers=layers, + ultra_relation_conditioning=conditioning)).load_pretrained(str(tmp_path / 'ultra.pth')).eval() + differences = {} + with torch.no_grad(), attached_context(model, context), evaluation_mode(model): + for h in range(context.num_entities): + for r in range(context.num_relations): + membership = port.project(torch.nn.functional.one_hot(torch.tensor(h), context.num_entities).float(), r) + differences[h, r] = (model.forward_k_vs_all(torch.tensor([[h, r]]))[0].sigmoid() - membership).abs().max().item() + direct = [d for (_, r), d in differences.items() if r in (0, 1)] + inverse = [d for (_, r), d in differences.items() if r in (2, 3)] + assert max(direct) < 1e-5 + assert (max(inverse) < 1e-5) == (conditioning == 'query') + with pytest.raises(ValueError, match='relation_conditioning'): + ULTRA(dict(num_entities=1, num_relations=1, ultra_relation_conditioning='inverse')) + + +@pytest.mark.parametrize('index', [0]) +@pytest.mark.parametrize('traversal', [False, True]) +def test_ultraquery_softmax_degree_calibration_follows_its_closed_form(tmp_path, index, traversal): + """Calibrated projections: min(d * softmax(logits), 0.9999), d the observed-tail mass reached from the fuzzy set.""" + from dicee.query_answering.methods._common import fuzzy_combine + fixture = torch.load(ROOT / 'ultra.pt', weights_only=True, map_location='cpu') + case = fixture['cases'][index] + context = QueryContext(fixture['triples'], fixture['num_entities'], fixture['num_relations']) + path = tmp_path / 'checkpoint.pt' + torch.save({'model_state_dict': case['state']}, path) + config = dict(case['config'], threshold=0.) + plain = load_method('ultraquery', path, context, **config) + model = load_method('ultraquery', path, context, **config, observed_traversal=traversal, calibration='softmax-degree') + assert model.provenance['configuration']['calibration'] == 'softmax-degree' + + def symbolic(source, relation): + result = torch.zeros_like(source) + for h, r, t in context.triples: + if r == relation: + result[t] = torch.maximum(result[t], source[h]) + return result + + def membership(tree): + if tree[0] == 'anchor': + return torch.nn.functional.one_hot(torch.tensor(tree[1]), context.num_entities).float() + if tree[0] == 'project': + source = membership(tree[2]) + logits = plain.project_batch(source[None], torch.tensor([tree[1]]), relation_ids=(tree[1],), logits=True)[0] + known = symbolic(source, tree[1]) + value = (logits.softmax(0) * known.sum().clamp_min(1)).clamp_max(.9999) + return torch.maximum(value, known) if traversal else value + if tree[0] == 'not': + return 1 - membership(tree[1]) + return fuzzy_combine([membership(child) for child in tree[1:]], tree[0], plain.logic) + + with torch.no_grad(): + for shape, query in fixture['queries'].items(): + tree = compile_query(query) + torch.testing.assert_close(model.membership(tree), membership(tree), atol=1e-6, rtol=1e-5, msg=shape) + # A single anchor head: d is its number of observed tails, as in the beam executor's calibration. + anchor, (relation,) = fixture['queries']['1p'] + tails = sorted(context.outgoing.get(anchor, {}).get(relation, ())) + logits = plain.project_batch(torch.nn.functional.one_hot(torch.tensor([anchor]), context.num_entities).float(), + torch.tensor([relation]), relation_ids=(relation,), logits=True)[0] + expected = (logits.softmax(0) * max(1, len(tails))).clamp_max(.9999) + if traversal: + expected[tails] = 1. + torch.testing.assert_close(model.membership(compile_query(fixture['queries']['1p'])), expected, atol=1e-6, rtol=1e-5) + with pytest.raises(ValueError, match='threshold'): + load_method('ultraquery', path, context, **dict(case['config'], threshold=.5), calibration='softmax-degree') + with pytest.raises(ValueError, match='calibration'): + load_method('ultraquery', path, context, **config, calibration='minmax') diff --git a/tests/test_query_benchmark.py b/tests/test_query_benchmark.py new file mode 100644 index 000000000..dbcf5daa0 --- /dev/null +++ b/tests/test_query_benchmark.py @@ -0,0 +1,384 @@ +"""Published graph layouts, hard-answer ranking, and shared benchmark execution.""" + +import io +import json +import os +import pickle +import zipfile +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pytest +import torch + +from dicee.query_answering import ( + BENCHMARK_DATASETS, + BenchmarkQuery, + QueryBenchmark, + QueryContext, + benchmark_model, + evaluate_benchmark, + load_benchmark, + summarize_benchmarks, +) +from dicee.query_answering._query import QUERY_SHAPES, ULTRAQUERY_SHAPES +from dicee.query_answering.benchmark import filtered_query_metrics +from dicee.query_answering.datasets import dataset_spec, download_benchmark +from tests.test_query_engine import TableModel, programs + + +def dump(path, name, value): + with (path / name).open('wb') as stream: + pickle.dump(value, stream) + + +def graph(path, name, direct, pairs, binary=False): + triples = list(QueryContext(direct, 8, 4, pairs).triples) + if binary: + torch.save(torch.tensor(triples), path / f'{name}.pt') + else: + (path / f'{name}.txt').write_text(''.join(f'{h}\t{r}\t{t}\n' for h, r, t in triples)) + return set(triples) + + +def fixture_dataset(root, name, binary=False): + group, folder, _ = dataset_spec(name) + path = root / folder + path.mkdir(parents=True) + pairs = ((0, 1), (2, 3)) if group == 'transductive' else ((0, 2), (1, 3)) + train = graph(path, 'train' if group == 'transductive' else 'train_graph', [(0, 0, 1), (1, 1 if group != 'transductive' else 2, 2)], pairs, binary) + test = None + if group == 'transductive': + dump(path, 'id2ent.pkl', {i: str(i) for i in range(8)}) + dump(path, 'id2rel.pkl', {i: str(i) for i in range(4)}) + (path / 'valid.txt').write_text('0\t0\t6\n') + (path / 'test.txt').write_text('0\t0\t7\n') + else: + graph(path, 'val_inference', [(2, 1, 5)], pairs, binary) + direct = [(2, 1, 3)] if group == 'inductive-e' else [(0, 1, 3), (1, 0, 2)] + test = graph(path, 'test_inference', direct, pairs, binary) + structured = {QUERY_SHAPES[shape]: [value[0]] for shape, value in programs(np.ones((4, 2, 4)) * .5, 2).items() + if shape in ULTRAQUERY_SHAPES} + for split in ('valid', 'test'): + hard_id = (5 if split == 'valid' else 3) if group == 'inductive-e' else (2 if split == 'valid' and group == 'inductive-er' else 3) + if group == 'transductive': + dump(path, f'{split}-queries.pkl', structured) + dump(path, f'{split}-easy-answers.pkl', {q: {1} for qs in structured.values() for q in qs}) + dump(path, f'{split}-hard-answers.pkl', {q: {hard_id} for qs in structured.values() for q in qs}) + else: + dump(path, f'{split}_queries.pkl', structured) + dump(path, f'{split}_answers_easy.pkl', {s: {q: {1} for q in qs} for s, qs in structured.items()}) + dump(path, f'{split}_answers_hard.pkl', {s: {q: {hard_id} for q in qs} for s, qs in structured.items()}) + return path, train, test + + +@pytest.mark.parametrize('name', ['FB15k237LogicalQuery', 'InductiveFB15k237Query:106', 'WikiTopicsQuery:art']) +@pytest.mark.parametrize('split', ['valid', 'test']) +@pytest.mark.parametrize('binary', [False, True]) +def test_official_layouts_no_target_leakage(tmp_path, name, split, binary): + path, train, test = fixture_dataset(tmp_path, name, binary) + data = load_benchmark(tmp_path, name, split=split) + assert len(data.queries) == 14 + assert {q.shape for q in data.queries} == set(ULTRAQUERY_SHAPES) + assert len(data.metadata['files']) >= 4 + edges = set(data.context.triples) + if data.group == 'transductive': + assert edges == train and data.candidates == tuple(range(8)) + assert not {'valid.txt', 'test.txt'} & data.metadata['files'].keys() + elif data.group == 'inductive-e': + assert data.context.num_entities == 6 + assert data.candidates == ((0, 1, 2, 5) if split == 'valid' else (0, 1, 2, 3)) + assert edges == (train | test) if split == 'test' else all(3 not in (h, t) for h, _, t in edges) + else: + assert edges == (train if split == 'valid' else test) + assert all(q.hard <= set(data.candidates) for q in data.queries) + selected = load_benchmark(tmp_path, name, split=split, query_types=['2u', 'pni']) + assert {q.shape for q in selected.queries} == {'2u', 'pni'} + + +def test_all_23_names_and_invalid_data(tmp_path): + assert len(BENCHMARK_DATASETS) == 23 + for name in BENCHMARK_DATASETS: + assert dataset_spec(name)[2].startswith('https://') + with pytest.raises(ValueError, match='Unknown benchmark'): + dataset_spec('WikiTopicsQuery:missing') + with pytest.raises(FileNotFoundError, match='extract'): + load_benchmark(tmp_path, 'FB15kLogicalQuery') + path, _, _ = fixture_dataset(tmp_path, 'FB15kLogicalQuery') + with pytest.raises(ValueError, match='distinct supported'): + load_benchmark(tmp_path, 'FB15kLogicalQuery', query_types=['2u-DNF']) + (path / 'train.txt').write_text('0\t0\t1\n') + with pytest.raises(ValueError, match='reciprocal'): + load_benchmark(tmp_path, 'FB15kLogicalQuery') + + +def test_extended_answers_keep_original_query_order(tmp_path): + path, _, _ = fixture_dataset(tmp_path, 'InductiveFB15k237Query:106') + shape = QUERY_SHAPES['1p'] + queries = [(2, (0,)), (0, (0,))] + dump(path, 'train_queries.pkl', {shape: queries}) + dump(path, 'train_answers_test.pkl', {shape: [{3}, {2}]}) + data = load_benchmark(tmp_path, 'InductiveFB15k237QueryExtendedEval:106', query_types=['1p']) + assert [q.hard for q in data.queries] == [frozenset({2}), frozenset({3})] + assert all(not q.easy for q in data.queries) + assert data.metadata['evaluation'] == 'faithfulness' + + +@pytest.mark.parametrize('tied', [False, True]) +def test_reference_sort_filter_protocol(tied): + # Independent implementation of the published positional rank adjustment. + scores = torch.tensor([.9, .8, .7, .6, .5, .4], dtype=torch.float64) + if tied: + scores[:] = .5 + easy, hard, candidates = {0}, {2, 4}, {0, 1, 2, 4, 5} + masked = scores.clone() + masked[3] = -torch.inf + order = masked.argsort(descending=True).tolist() + answer_order = [i for i in order if i in easy | hard] + ranks = [order.index(h) - answer_order.index(h) + 1 for h in sorted(hard)] + result = filtered_query_metrics(scores, easy, hard, candidates=candidates) + assert result['mrr'] == pytest.approx(np.mean([1 / r for r in ranks])) + assert result['hits1'] == pytest.approx(np.mean([r == 1 for r in ranks])) + + +def test_sort_excludes_outside_candidates_tied_at_negative_infinity(): + # The released UltraQuery evaluator assigns rank 3 here because node 1, + # outside the inductive candidate domain, ties with the hard answer. + scores = torch.full((6,), -torch.inf) + result = filtered_query_metrics(scores, set(), {2}, candidates=[0, 2, 4]) + assert result['mrr'] == .5 + assert result['hits1'] == 0 + + +@pytest.mark.parametrize('policy,rank', [('average', 2.), ('optimistic', 1.), ('pessimistic', 3.)]) +def test_ties_and_negative_infinity_exclude_other_answers(policy, rank): + scores = torch.full((6,), -torch.inf) + result = filtered_query_metrics(scores, {0}, {1, 2}, candidates=[0, 1, 2, 3, 4], tie_policy=policy) + assert result['mrr'] == pytest.approx(1 / rank) + with pytest.raises(ValueError, match='disjoint'): + filtered_query_metrics(scores, {0}, {0}) + with pytest.raises(ValueError, match='score vector'): + filtered_query_metrics(torch.tensor([float('nan')]), set(), {0}) + + +def test_query_then_shape_then_dataset_averaging_and_partial_flags(): + context = QueryContext([(0, 0, 1)], 4, 1) + queries = (BenchmarkQuery('1p', (0, (0,)), frozenset({0}), frozenset({1, 2})), + BenchmarkQuery('1p', (1, (0,)), frozenset(), frozenset({3})), + BenchmarkQuery('2p', (0, (0, 0)), frozenset(), frozenset({0}))) + data = QueryBenchmark('fixture', 'transductive', 'test', context, queries, tuple(range(4))) + scores = torch.tensor([4., 3., 2., 1.]) + report = evaluate_benchmark(data, lambda q: scores) + assert report['per_shape']['1p']['mrr'] == pytest.approx((1 + .25) / 2) + assert report['averages']['epfo']['mrr'] == pytest.approx(((1 + .25) / 2 + 1) / 2) + subset = evaluate_benchmark(data, lambda q: scores, max_queries_per_shape=1) + assert subset['queries'] == 2 and not subset['protocol']['full_split'] + assert not subset['protocol']['all_14_shapes'] + other = dict(subset, dataset='other') + summary = summarize_benchmarks([report, other]) + assert summary['groups']['all']['epfo']['mrr'] == pytest.approx((report['averages']['epfo']['mrr'] + 1) / 2) + assert not summary['complete_23_dataset_test_suite'] + with pytest.raises(ValueError, match='Duplicate'): + summarize_benchmarks([report, report]) + with pytest.raises(ValueError, match='complete public'): + evaluate_benchmark(data, lambda q: scores[:2]) + + +@pytest.mark.parametrize('name', ['ULTRA', 'TRIX', 'Flock']) +@pytest.mark.parametrize('attached', [False, True]) +def test_real_models_restore_graph_modes_and_failure(tmp_path, name, attached): + from dicee.models import TRIX, ULTRA, Flock + fixture_dataset(tmp_path, 'WikiTopicsQuery:art') + data = load_benchmark(tmp_path, 'WikiTopicsQuery:art') + settings = dict(num_entities=3, num_relations=2, ultra_dim=8, ultra_num_layers=2, trix_dim=8, + flock_dim=8, flock_walk_num=2, flock_walk_len=8, flock_refinements=2, flock_seed=77) + model = {'ULTRA': ULTRA, 'TRIX': TRIX, 'Flock': Flock}[name](settings) + if attached: + model.set_graph(torch.tensor([[0, 0, 1]]), inverse_relations={0: 1}) + original = QueryContext.from_model(model) if attached else None + states = {k: v.clone() for k, v in model.state_dict().items()} + report = benchmark_model(model, data, beam_size=2, seed=42) + assert report['queries'] == 14 and report['protocol']['all_14_shapes'] + assert report['inference']['method'] == 'cqd-global-prefix' + assert model.training and (model.num_entities, model.num_relations) == (3, 2) + for key, value in model.state_dict().items(): + torch.testing.assert_close(value, states[key], rtol=0, atol=0) + assert QueryContext.from_model(model) == original if attached else model.graph_triples is None + with pytest.raises(ValueError, match='positive integer'): + benchmark_model(model, data, beam_size=0) + assert QueryContext.from_model(model) == original if attached else model.graph_triples is None + second = benchmark_model(model, data, beam_size=2, seed=42) + assert second['per_shape'] == report['per_shape'] + + +@pytest.mark.parametrize('executor', ['cqd', 'qto']) +def test_cli_checkpoint_reports_and_preserves_commit_work(tmp_path, monkeypatch, executor): + from dicee.models import ULTRA + from dicee.query_answering.__main__ import main + fixture_dataset(tmp_path, 'FB15k237LogicalQuery') + fixture_dataset(tmp_path, 'WikiTopicsQuery:art') + config = dict(ultra_dim=8, ultra_num_layers=2) + model = ULTRA(dict(config, num_entities=1, num_relations=1)) + checkpoint, config_path, output = tmp_path / 'model.pt', tmp_path / 'config.json', tmp_path / 'report.json' + torch.save({'model': model.state_dict()}, checkpoint) + config_path.write_text(json.dumps(config)) + monkeypatch.setattr('sys.argv', ['query_answering', 'benchmark', '--data-root', str(tmp_path), + '--datasets', 'FB15k237LogicalQuery', 'WikiTopicsQuery:art', + '--checkpoint', str(checkpoint), '--model-config', str(config_path), + '--output', str(output), '--beam-size', '2', '--max-queries-per-shape', '1', + '--query-sampling', 'uniform', '--sampling-seed', '83', '--executor', executor]) + main() + report = json.loads(output.read_text()) + assert len(report['results']) == 2 + assert all(r['inference']['executor'] == executor for r in report['results']) + assert all(r['protocol']['query_sampling'] == 'uniform' and r['protocol']['sampling_seed'] == 83 for r in report['results']) + assert report['summary']['groups']['all']['datasets'] == 2 + assert not report['summary']['complete_23_dataset_test_suite'] + assert report['results'][0]['inference']['backbone_state_sha256'] == report['results'][1]['inference']['backbone_state_sha256'] + monkeypatch.setattr('dicee.query_answering.engine.QueryAnswerer.predict', lambda *a, **k: pytest.fail('Completed run was rescored')) + main() + resumed = json.loads(output.read_text()) + assert resumed['state'] == 'complete' + assert [r['per_shape'] for r in resumed['results']] == [r['per_shape'] for r in report['results']] + + +def test_download_archive_paths(tmp_path, monkeypatch): + memory = io.BytesIO() + with zipfile.ZipFile(memory, 'w') as archive: + archive.writestr('../escape.txt', 'bad') + monkeypatch.setattr('urllib.request.urlopen', lambda *a, **k: io.BytesIO(memory.getvalue())) + with pytest.raises(ValueError, match='Unsafe path'): + download_benchmark('WikiTopicsQuery:art', tmp_path) + assert not (tmp_path.parent / 'escape.txt').exists() + + +def test_prediction_receives_only_query_and_candidate_filtering_is_final(tmp_path): + fixture_dataset(tmp_path, 'InductiveFB15k237Query:106') + data = load_benchmark(tmp_path, 'InductiveFB15k237Query:106', query_types=['2p']) + # Entity 5 is outside the final candidate domain but remains a valid + # intermediate entity in the complete public vocabulary. + raw = torch.full((6, 4, 6), -20., dtype=torch.float64) + raw[0, 0, 5] = 10 + raw[5, 1, 3] = 10 + query = replace(data.queries[0], query=(0, (0, 1)), easy=frozenset(), hard=frozenset({3})) + data.queries = (query,) + result = benchmark_model(TableModel(raw), data, beam_size=1) + assert result['per_shape']['2p']['mrr'] == 1. + + +def test_wikitopics_unused_relations_use_original_mapping(tmp_path): + path, _, _ = fixture_dataset(tmp_path, 'WikiTopicsQuery:art') + # Full vocabulary has six relation IDs, but the last inverse never appears. + edges = QueryContext([(0, 0, 1), (2, 1, 3)], 4, 5, ((0, 3), (1, 4))).triples + (path / 'test_inference.txt').write_text(''.join(f'{h}\t{r}\t{t}\n' for h, r, t in edges)) + dump(path, 'og_mappings.pkl', {'r2id': {'a': 0, 'b': 1, 'unused': 2, 'a_inv': 3, 'b_inv': 4, 'unused_inv': 5}}) + data = load_benchmark(tmp_path, 'WikiTopicsQuery:art') + assert data.context.num_relations == 5 + assert data.context.inverse_relations == ((0, 3), (1, 4)) + assert data.context.triples == edges + assert 'og_mappings.pkl' in data.metadata['files'] + + +def test_cli_rejects_mismatched_transductive_vocabulary(tmp_path, monkeypatch): + from dicee.models import DistMult + from dicee.query_answering.__main__ import main + from tests.test_query_engine import save_experiment + path, _, _ = fixture_dataset(tmp_path, 'FB15k237LogicalQuery') + settings = dict(model='DistMult', num_entities=8, num_relations=4, embedding_dim=8) + model = DistMult(settings) + experiment = tmp_path / 'experiment' + save_experiment(model, settings, experiment) + output = tmp_path / 'results.json' + monkeypatch.setattr('sys.argv', ['query_answering', 'benchmark', '--data-root', str(tmp_path), + '--experiment', str(experiment), '--output', str(output), '--query-types', '1p']) + main() + assert output.is_file() + dump(path, 'id2ent.pkl', {i: str((i + 1) % 8) for i in range(8)}) + with pytest.raises(ValueError, match='exactly match'): + main() + + +def test_suite_complete_only_with_all_datasets_shapes_queries(): + base = dict(split='test', protocol=dict(full_split=True, all_14_shapes=True), + dataset_metadata={'evaluation': 'hard-answers'}, + averages={category: dict.fromkeys(('mrr', 'hits1', 'hits3', 'hits10'), .5) + for category in ('all', 'epfo', 'negation')}) + reports = [dict(base, dataset=name, group=dataset_spec(name)[0]) for name in BENCHMARK_DATASETS] + assert summarize_benchmarks(reports)['complete_23_dataset_test_suite'] + assert not summarize_benchmarks(reports[:-1])['complete_23_dataset_test_suite'] + reports[0] = dict(reports[0], dataset_metadata={'evaluation': 'faithfulness'}) + with pytest.raises(ValueError, match='faithfulness'): + summarize_benchmarks(reports) + + +@pytest.mark.parametrize('device', ['cpu', pytest.param('cuda', marks=pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA unavailable'))]) +@pytest.mark.parametrize('restricted', [False, True]) +def test_reused_ranking_masks_match_independent_sort_oracle(device, restricted): + from dicee.query_answering.benchmark import QueryMetrics + n = 29 + candidates = set(range(n)) - ({0, 11, 24} if restricted else set()) + evaluator = QueryMetrics(n, candidates=candidates) + rng = np.random.default_rng(13) + for tied in (False, True): + for _ in range(20): + scores = torch.tensor(rng.integers(0, 4, n) if tied else rng.normal(size=n), device=device, dtype=torch.float64) + if tied: + scores[scores == 0] = -torch.inf + unchanged = scores.clone() + labels = rng.choice(sorted(candidates), 6, replace=False).tolist() + easy, hard = set(labels[:3]), set(labels[3:]) + masked = scores.clone() + masked[list(set(range(n)) - candidates)] = -torch.inf + order = masked.argsort(descending=True).tolist() + ranks = [1 + sum(e in candidates and e not in easy | hard for e in order[:order.index(t)]) for t in hard] + expected = dict(mrr=np.mean([1/r for r in ranks]), hits1=np.mean([r <= 1 for r in ranks]), + hits3=np.mean([r <= 3 for r in ranks]), hits10=np.mean([r <= 10 for r in ranks])) + assert evaluator(scores, easy, hard) == pytest.approx(expected) + assert torch.equal(scores, unchanged) + + +@pytest.mark.integration +@pytest.mark.skipif(not os.environ.get('DICEE_ULTRAQUERY_DATA_ROOT'), reason='Set DICEE_ULTRAQUERY_DATA_ROOT to the official extracted data') +@pytest.mark.parametrize('name', ['FB15k237LogicalQuery', 'InductiveFB15k237Query:106', 'WikiTopicsQuery:art']) +@pytest.mark.parametrize('split', ['valid', 'test']) +def test_official_ultraquery_data_graphs_and_reference_ranks(name, split): + """Real-data conformance for each of the three pinned UltraQuery loaders.""" + root = Path(os.environ['DICEE_ULTRAQUERY_DATA_ROOT']) + data = load_benchmark(root, name, split=split) + path = root / dataset_spec(name)[1] + + def triples(stem): + file = path / f'{stem}.txt' + if file.is_file(): + return {tuple(map(int, row.split())) for row in file.read_text().splitlines() if row.strip()} + return {tuple(row) for row in torch.load(path / f'{stem}.pt', weights_only=True).tolist()} + + if data.group == 'transductive': + observed = triples('train') + assert data.candidates == tuple(range(data.context.num_entities)) + elif data.group == 'inductive-e': + observed = triples('train_graph') | triples('val_inference' if split == 'valid' else 'test_inference') + else: + observed = triples('train_graph' if split == 'valid' else 'test_inference') + assert set(data.context.triples) == observed + if data.group != 'transductive': + assert set(data.candidates) == {entity for h, _, t in observed for entity in (h, t)} + scores = torch.sin(torch.arange(data.context.num_entities, dtype=torch.float64)) + report = evaluate_benchmark(data, lambda q: scores, max_queries_per_shape=1) + first = {} + for query in data.queries: + first.setdefault(query.shape, query) + masked = scores.clone() + masked[list(set(range(len(scores))) - set(data.candidates))] = -torch.inf + order = masked.argsort(descending=True).tolist() + positions = {entity: i for i, entity in enumerate(order)} + for shape, query in first.items(): + true_answers = sorted(query.easy | query.hard, key=positions.__getitem__) + ranks = [positions[entity] - offset + 1 for offset, entity in enumerate(true_answers) if entity in query.hard] + expected = dict(mrr=np.mean([1 / r for r in ranks]), + **{f'hits{k}': np.mean([r <= k for r in ranks]) for k in (1, 3, 10)}) + assert {key: report['per_shape'][shape][key] for key in expected} == pytest.approx(expected) + assert report['queries'] == 14 and report['protocol']['all_14_shapes'] + assert not report['protocol']['full_split'] + diff --git a/tests/test_query_engine.py b/tests/test_query_engine.py new file mode 100644 index 000000000..a1f270a62 --- /dev/null +++ b/tests/test_query_engine.py @@ -0,0 +1,646 @@ +"""Independent fuzzy-logic examples and common KGE/KGFM query contracts.""" + +import hashlib +import json + +import numpy as np +import pytest +import torch +from torch import nn + +from dicee import KGE +from dicee.query_answering import QueryAnswerer, QueryContext, QueryScoreAdapter +from dicee.query_answering._query import QUERY_SHAPES, compile_query, index_query +from dicee.query_answering.context import state_fingerprint +from dicee.query_answering.score_adapter import score_features + + +class TableModel(nn.Module): + name = 'Table' + + def __init__(self, raw): + super().__init__() + self.table = nn.Parameter(torch.as_tensor(raw, dtype=torch.float64).clone()) + self.num_entities, self.num_relations = self.table.shape[:2] + self.calls = [] + + def forward_k_vs_all(self, pairs): + self.calls.append(pairs.detach().cpu().tolist()) + return self.table[pairs[:, 0], pairs[:, 1]] + + +def programs(m, beam, tnorm='prod'): + """Literal formulas, independent of the parser and recursive evaluator.""" + n = len(m) + a, b, c = (0, (0,)), (1, (1,)), (2, (0,)) + pa, pb, pc = m[0, 0], m[1, 1], m[2, 0] + conjunction = np.multiply if tnorm == 'prod' else np.minimum + union = (lambda x, y: x + y - x*y) if tnorm == 'prod' else np.maximum + + def project(prefix, r): + heads = sorted(range(n), key=lambda h: (-prefix[h], h))[:beam] + return np.array([max(conjunction(prefix[h], m[h, r, t]) for h in heads) for t in range(n)]) + + two = project(pa, 1) + intersection, disjunction = (a, b), (a, b, (-1,)) + return { + '1p': (a, pa), '2p': ((0, (0, 1)), two), '3p': ((0, (0, 1, 0)), project(two, 0)), + '2i': (intersection, conjunction(pa, pb)), '3i': ((a, b, c), conjunction(conjunction(pa, pb), pc)), + 'pi': (((0, (0, 1)), b), conjunction(two, pb)), + 'ip': ((intersection, (1,)), project(conjunction(pa, pb), 1)), + '2u': (disjunction, union(pa, pb)), 'up': ((disjunction, (1,)), project(union(pa, pb), 1)), + '2in': ((a, (1, (1, -2))), conjunction(pa, 1-pb)), + '3in': ((a, c, (1, (1, -2))), conjunction(conjunction(pa, pc), 1-pb)), + 'inp': (((a, (1, (1, -2))), (1,)), project(conjunction(pa, 1-pb), 1)), + 'pin': (((0, (0, 1)), (1, (1, -2))), conjunction(two, 1-pb)), + 'pni': (((0, (0, 1, -2)), b), conjunction(1-two, pb)), + '4p': ((0, (0, 1, 0, 1)), project(project(two, 0), 1)), + '4i': ((a, b, c, (3, (1,))), conjunction(conjunction(conjunction(pa, pb), pc), m[3, 1])), + } + + +@pytest.fixture +def raw(): + return np.random.default_rng(17).normal(size=(4, 2, 4)) + + +@pytest.mark.parametrize('shape', QUERY_SHAPES) +@pytest.mark.parametrize('beam', [1, 2, 4]) +@pytest.mark.parametrize('tnorm', ['prod', 'min']) +def test_all_shapes_against_literal_formulas(raw, shape, beam, tnorm): + query, expected = programs(1/(1+np.exp(-raw)), beam, tnorm)[shape] + model = TableModel(raw) + result = QueryAnswerer(model).predict(query, beam_size=beam, tnorm=tnorm) + np.testing.assert_allclose(result.numpy(), expected, atol=2e-15, rtol=2e-14) + assert model.training # prediction restores its caller's mode + assert model.table.grad is None + + +def named(query): + if isinstance(query, tuple): + return tuple(named(x) for x in query) + return 'not' if query == -2 else 'union' if query == -1 else str(query) + + +def wrapper(model): + kge = object.__new__(KGE) + kge.model = model + # Insertion order deliberately differs from ID order. + kge.entity_to_idx = {str(i): i for i in reversed(range(model.num_entities))} + kge.relation_to_idx = {str(i): i for i in range(model.num_relations)} + return kge + + +@pytest.mark.parametrize('shape', QUERY_SHAPES) +def test_named_api_limits_batches_and_numeric_ids(raw, shape): + query, expected = programs(1/(1+np.exp(-raw)), 2)[shape] + kge = wrapper(TableModel(raw)) + kwargs = dict(query_type=shape, beam_size=2, k=1) + result = kge.answer_multi_hop_query(query=named(query), **kwargs) + assert len(result) == 1 and result[0][0] == str(int(np.argmax(expected))) + scores = kge.answer_multi_hop_query(query=named(query), only_scores=True, **kwargs) + np.testing.assert_allclose(scores.numpy(), expected, atol=2e-15, rtol=2e-14) + batch = kge.answer_multi_hop_query(queries=[named(query)] * 2, **kwargs) + assert batch == [result, result] + assert kge.answer_multi_hop_query(query=named(query), query_type=shape, k=0) == [] + + +def test_singleton_ties_invalid_queries_and_logits(raw): + kge = wrapper(TableModel(np.zeros((1, 1, 1)))) + assert kge.answer_multi_hop_query('2p', ('0', ('0', '0')))[0][0] == '0' + engine = QueryAnswerer(TableModel(np.zeros_like(raw))) + scores = engine.predict((0, (0, 1)), beam_size=1) + assert torch.equal(scores, torch.full_like(scores, .25)) + assert engine.scorer.model.calls[1] == [[0, 1]] + with pytest.raises(ValueError, match='Anchor'): + engine.predict((4, (0,))) + with pytest.raises(ValueError, match='Relation'): + engine.predict((0, (2,))) + with pytest.raises(ValueError, match='lambda'): + engine.predict((0, (0, -2)), neg_norm='yager') + with pytest.raises(ValueError, match='beam_size'): + engine.predict((0, (0,)), beam_size=0) + with pytest.raises(ValueError, match='marker'): + index_query('2in', (('0', ('0',)), ('1', ('1', 'wrong'))), {'0': 0, '1': 1}, {'0': 0, '1': 1}) + raw_engine = QueryAnswerer(TableModel(raw)) + actual = raw_engine.predict(((0, (0,)), (1, (1,))), use_logits=True) + np.testing.assert_allclose(actual.numpy(), raw[0, 0] * raw[1, 1]) + with pytest.raises(ValueError, match='Raw logits'): + QueryAnswerer(TableModel(raw), adapter=QueryScoreAdapter('global')).predict((0, (0,)), use_logits=True) + + +@pytest.mark.parametrize('shape', ['2u', 'up']) +def test_named_unions_accept_marker_omission(raw, shape): + query = (('0', ('0',)), ('1', ('1',))) + marked = (*query, ('union',)) + if shape == 'up': + query, marked = (query, ('0',)), (marked, ('0',)) + kge = wrapper(TableModel(raw)) + torch.testing.assert_close(kge.answer_multi_hop_query(shape, query, only_scores=True), + kge.answer_multi_hop_query(shape, marked, only_scores=True)) + + +@pytest.mark.parametrize('norm,parameter', [('standard', 0.), ('sugeno', .5), ('yager', 2.)]) +def test_negation_norms(raw, norm, parameter): + p = 1 / (1 + np.exp(-raw[0, 0])) + expected = 1-p if norm == 'standard' else (1-p)/(1+parameter*p) if norm == 'sugeno' else (1-p**parameter)**(1/parameter) + actual = QueryAnswerer(TableModel(raw)).predict((0, (0, -2)), neg_norm=norm, lambda_=parameter) + np.testing.assert_allclose(actual.numpy(), expected) + + +def test_observed_proofs_survive_pruning_and_do_not_shortcut_negation(): + raw = np.full((4, 2, 4), -8.) + raw[0, 0, 0] = 8 + context = QueryContext([(0, 0, 1), (0, 0, 2), (2, 1, 3)], 4, 2) + engine = QueryAnswerer(TableModel(raw), context=context, observed_mix=1) + result = engine.predict((0, (0, 1)), beam_size=1) + # Stable top-1 visits entity 1; the proof via entity 2 is restored separately. + assert result[3] == 1 and engine.last_info['pruned'] + negated = engine.predict((0, (0, 1, -2)), beam_size=1) + assert negated[3] > .99 and engine.last_info['negated_pruning'] + exact = engine.predict((0, (0, 1, -2)), beam_size=4) + assert exact[3] == 0 + + +def test_negation_from_known_facts_excludes_only_their_answers(raw): + # Known answers: (1, 1) -> {2, 3}; the chain 0 -0-> 1 -1-> reaches {2, 3} too. + context = QueryContext([(0, 0, 1), (1, 1, 2), (1, 1, 3)], 4, 2) + engine = QueryAnswerer(TableModel(raw), context=context, observed_mix=1) + keep = torch.tensor([1., 1., 0., 0.], dtype=torch.float64) + pa, pb = engine.predict((0, (0,))), engine.predict((1, (1,))) + two_in = ((0, (0,)), (1, (1, -2))) + torch.testing.assert_close(engine.predict(two_in, negation='observed'), pa * keep, rtol=1e-14, atol=0) + # The default still negates the backbone's memberships, not the facts. + torch.testing.assert_close(engine.predict(two_in), pa * (1 - pb), rtol=1e-14, atol=0) + pni = ((0, (0, 1, -2)), (1, (1,))) + torch.testing.assert_close(engine.predict(pni, negation='observed'), pb * keep, rtol=1e-14, atol=0) + with pytest.raises(ValueError, match='Negation must be'): + engine.predict(two_in, negation='facts') + with pytest.raises(ValueError, match='known facts needs'): + QueryAnswerer(TableModel(raw)).predict(two_in, negation='observed') + + +def test_cache_bounded_reused_and_invalidated(raw): + model = TableModel(raw) + engine = QueryAnswerer(model, cache_bytes=4*8) + a = (0, (0,)) + before = engine.predict((a, a)) + assert engine.last_info['raw_rows'] == 1 and engine.last_info['cache_hits'] == 1 + assert engine.last_info['cache_bytes'] <= 32 + with torch.no_grad(): + model.table.add_(1) + after = engine.predict((a, a)) + assert (after > before).all() + engine.predict((0, (0, 1)), beam_size=4) + assert engine.last_info['cache_bytes'] <= 32 + + +@pytest.mark.parametrize('name', ['DistMult', 'ULTRA', 'TRIX', 'Flock']) +def test_saved_experiment_query_reload(name, tmp_path): + from dicee.models import TRIX, ULTRA, DistMult, Flock + cls = {'DistMult': DistMult, 'ULTRA': ULTRA, 'TRIX': TRIX, 'Flock': Flock}[name] + settings = dict(model=name, num_entities=4, num_relations=2, embedding_dim=8, + ultra_dim=8, ultra_num_layers=2, trix_dim=8, flock_dim=8, + flock_walk_num=2, flock_walk_len=8, flock_refinements=2, flock_seed=17) + model = cls(settings) + if name != 'DistMult': + model.set_graph(torch.tensor([[0, 0, 1], [1, 1, 2], [2, 0, 3]])) + save_experiment(model, settings, tmp_path) + restored = KGE(path=str(tmp_path)) + actual = restored.answer_multi_hop_query('3p', ('0', ('0', '1', '0')), k=2, only_scores=True) + expected = QueryAnswerer(model).predict((0, (0, 1, 0)), beam_size=2) + torch.testing.assert_close(actual, expected) + + +def save_experiment(model, settings, path): + path.mkdir(parents=True, exist_ok=True) + if hasattr(model, 'save_graph'): + model.save_graph(path/model.graph_filename) + torch.save(model.state_dict(), path/'model.pt') + (path/'configuration.json').write_text(json.dumps(settings)) + (path/'report.json').write_text(json.dumps(dict(num_entities=model.num_entities, num_relations=model.num_relations))) + for name, count in [('entity', model.num_entities), ('relation', model.num_relations)]: + (path/f'{name}_to_idx.csv').write_text(f',{name}\n' + ''.join(f'{i},{i}\n' for i in range(count))) + + +def test_generator_and_context_truth_against_boolean_oracle(raw): + from dicee.query_generator import QueryGenerator + facts = [(h, r, t) for h, r, t in np.ndindex(raw.shape) if raw[h, r, t] > 0] + context = QueryContext(facts, 4, 2) + generator = QueryGenerator.from_context(context) + for query, expected in programs((raw > 0).astype(float), 4).values(): + answers = set(np.flatnonzero(expected).tolist()) + assert context.answers(compile_query(query)) == answers + assert generator.achieve_answer(generator.tuple2list(query), generator.ent_in, generator.ent_out) == answers + + +def test_mixed_module_modes_and_scoring_failure_are_restored(raw, monkeypatch): + model = TableModel(raw) + model.add_module('dropout', nn.Dropout()) + model.train() + model.dropout.eval() + engine = QueryAnswerer(model) + engine.predict((0, (0,))) + assert model.training and not model.dropout.training + + def fail(_): + raise RuntimeError('scoring failed') + monkeypatch.setattr(model, 'forward_k_vs_all', fail) + with pytest.raises(RuntimeError, match='scoring failed'): + engine.predict((0, (0,))) + assert model.training and not model.dropout.training + + +@pytest.mark.parametrize('mode', ['global', 'context', 'context_scores_v1']) +@pytest.mark.parametrize('gamma', [0., .4, 1.]) +def test_adapter_against_independent_numpy_formulas(raw, mode, gamma): + context = QueryContext([(0, 0, 1), (0, 0, 2), (1, 1, 3)], 4, 2) + conditions = [(0, 0), (1, 1)] + z = np.stack([raw[h, r] for h, r in conditions]) + observed = np.zeros_like(z, dtype=bool) + observed[0, [1, 2]], observed[1, 3] = True, True + base = np.array([[1, np.log1p(2)/np.log1p(4), np.log1p(2)/np.log1p(8), np.log1p(2)/np.log1p(16)], + [1, np.log1p(1)/np.log1p(4), np.log1p(1)/np.log1p(8), np.log1p(1)/np.log1p(16)]]) + features = base[:, :1] if mode == 'global' else base + if mode == 'context_scores_v1': + prob = np.exp(z - z.max(1, keepdims=True)) + prob /= prob.sum(1, keepdims=True) + top = np.sort(z, axis=1)[:, -2:] + features = np.column_stack((base, np.tanh(z.mean(1)/4), -(prob*np.log(prob)).sum(1)/np.log(4), + np.tanh((top[:, 1]-top[:, 0])/4), + np.tanh(((z*observed).sum(1)/observed.sum(1)-z.mean(1))/4))) + weights = np.random.default_rng(33).normal(size=(2, features.shape[1])) + logits = 2**np.tanh(features @ weights[0])[:, None]*z + 4*np.tanh(features @ weights[1])[:, None] + expected = 1/(1+np.exp(-logits)) + expected[observed] = gamma + (1-gamma)*expected[observed] + obs, actual_base = context.features(conditions, device='cpu') + np.testing.assert_allclose(actual_base.numpy(), base) + np.testing.assert_allclose(score_features(torch.tensor(z), obs, actual_base, mode).numpy(), features, atol=2e-15) + actual = QueryScoreAdapter(mode, gamma, weights=weights)(torch.tensor(z), obs, actual_base) + np.testing.assert_allclose(actual.detach().exp().numpy(), expected, atol=2e-15) + + +def test_adapter_artifacts_bind_actual_weights(raw, tmp_path): + model = TableModel(raw) + adapter = QueryScoreAdapter('global', metadata={'backbone_state_sha256': state_fingerprint(model)}) + path = tmp_path/'adapter.json' + adapter.save(path) + loaded = QueryScoreAdapter.load(path, model=model) + torch.testing.assert_close(adapter.weights, loaded.weights) + checkpoint = tmp_path/'model.pt' + torch.save({'model': model.state_dict()}, checkpoint) + catalog = {'all_sources': dict(weights=[[0.]*4]*2, feature_mode='context', observed_mix=1., + backbone_checkpoint_sha256=hashlib.sha256(checkpoint.read_bytes()).hexdigest())} + path.write_text(json.dumps(catalog)) + QueryScoreAdapter.load(path, model=model, key='all_sources', checkpoint=checkpoint) + with pytest.raises(ValueError, match='original checkpoint'): + QueryScoreAdapter.load(path, model=model, key='all_sources') + with torch.no_grad(): + model.table[0, 0, 0] += 1 + with pytest.raises(ValueError, match='modified backbone'): + QueryAnswerer(model, adapter=loaded).predict((0, (0,))) + with pytest.raises(ValueError, match='modified backbone'): + QueryScoreAdapter.load(path, model=model, key='all_sources', checkpoint=checkpoint) + + +@pytest.mark.parametrize('name', ['ULTRA', 'TRIX', 'Flock']) +def test_actual_kgfms_share_the_api_and_restore_context(name): + from dicee.models import TRIX, ULTRA, Flock + cls = {'ULTRA': ULTRA, 'TRIX': TRIX, 'Flock': Flock}[name] + settings = dict(num_entities=4, num_relations=4, ultra_dim=8, ultra_num_layers=2, trix_dim=8, + flock_dim=8, flock_walk_num=2, flock_walk_len=8, flock_refinements=2, flock_seed=77) + facts = torch.tensor([[0, 2, 1], [1, 0, 2], [2, 2, 3]]) + model = cls(settings).set_graph(facts, inverse_relations={2: 1, 0: 3}).eval() + context = QueryContext.from_model(model) + assert context.inverse_relations == ((0, 3), (2, 1)) + assert (1, 1, 0) in context.triples and (2, 3, 1) in context.triples + engine = QueryAnswerer(model, observed_mix=1, seed=19) + first = engine.predict((0, (2, 0)), beam_size=2) + assert first[2] == 1 and first.shape == (4,) + kge = wrapper(model) + for shape in QUERY_SHAPES: + query = programs(np.full((4, 2, 4), .5), 2)[shape][0] + result = kge.answer_multi_hop_query(shape, named(query), only_scores=True, k=2) + assert result.shape == (4,) and torch.isfinite(result).all() + if name == 'Flock': + from dicee.query_answering.engine import AtomicScorer + state = torch.random.get_rng_state().clone() + scorer = AtomicScorer(model, seed=9) + batched = scorer.rows([(0, 2), (1, 0)]) + serial = torch.cat([scorer.rows([(h, r)]) for h, r in [(1, 0), (0, 2)]]) + torch.testing.assert_close(batched, serial.flip(0), rtol=0, atol=0) + assert torch.equal(state, torch.random.get_rng_state()) and model.seed == 77 + model.set_graph(torch.tensor([[0, 2, 3]]), inverse_relations={2: 1, 0: 3}) + with pytest.raises(ValueError, match='must match'): + QueryAnswerer(model, context=context) + after = engine.predict((0, (2,)), beam_size=2) + assert after[3] == 1 and engine.scorer.context != context + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA unavailable') +def test_cuda_scores_stay_on_device(raw): + model = TableModel(raw).cuda() + engine = QueryAnswerer(model) + result = engine.predict((0, (0, 1))) + assert result.is_cuda + expected = QueryAnswerer(TableModel(raw)).predict((0, (0, 1))) + torch.testing.assert_close(result.cpu(), expected) + + +@pytest.mark.parametrize('shape', QUERY_SHAPES) +@pytest.mark.parametrize('tnorm', ['prod', 'min']) +def test_persistent_cache_matches_formulas_and_owns_returned_scores(raw, shape, tnorm): + query, expected = programs(1/(1+np.exp(-raw)), 2, tnorm)[shape] + model = TableModel(raw).eval() + engine = QueryAnswerer(model, row_batch_size=2) + first = engine.predict(query, beam_size=2, tnorm=tnorm, return_log_scores=True) + np.testing.assert_allclose(first.exp().numpy(), expected, atol=2e-15, rtol=2e-14) + calls = len(model.calls) + first.fill_(123.) + second = engine.predict(query, beam_size=2, tnorm=tnorm) + np.testing.assert_allclose(second.numpy(), expected, atol=2e-15, rtol=2e-14) + assert len(model.calls) == calls and engine.last_info['raw_rows'] == 0 + assert engine.last_info['cache_hits'] > 0 + + +def test_persistent_lru_eviction_resize_and_disable(raw): + model = TableModel(raw).eval() + engine = QueryAnswerer(model, cache_bytes=64) + for head in (0, 1, 0, 2, 0): + engine.predict((head, (0,))) + assert len(model.calls) == 3 + engine.predict((1, (0,))) + assert engine.last_info['raw_rows'] == 1 + engine.cache_bytes = 32 + engine.predict((1, (0,))) + assert engine.last_info['raw_rows'] == 0 and engine.last_info['cache_bytes'] == 32 + engine.cache_bytes = 0 + engine.predict((1, (0,))) + assert engine.last_info['raw_rows'] == 1 and engine.last_info['cache_bytes'] == 0 + # Eviction during a beam batch must not invalidate its in-flight rows. + engine.cache_bytes = 32 + actual = engine.predict((0, (0, 1)), beam_size=4) + torch.testing.assert_close(actual, QueryAnswerer(TableModel(raw), cache_bytes=0).predict((0, (0, 1)), beam_size=4)) + engine.clear_cache() + assert not engine._cache and engine._cache_used == 0 + + +@pytest.mark.parametrize('change', ['inplace', 'parameter', 'buffer', 'load', 'dtype', 'adapter', 'observed', 'context', 'precision']) +def test_persistent_cache_invalidates_changed_inputs(raw, change, monkeypatch): + model = TableModel(raw).eval() + model.register_buffer('extra', torch.tensor(0.)) + context = QueryContext([(0, 0, 1)], 4, 2) + adapter = QueryScoreAdapter('context', observed_mix=.5) + engine = QueryAnswerer(model, adapter=adapter, context=context) + query = (0, (0,)) + engine.predict(query) + engine.predict(query) + assert engine.last_info['raw_rows'] == 0 + with torch.no_grad(): + if change == 'inplace': + model.table.add_(1) + elif change == 'parameter': + model.table = nn.Parameter(model.table.clone() + 1) + elif change == 'buffer': + model.extra.add_(1) + elif change == 'load': + model.load_state_dict(dict(table=model.table + 1, extra=model.extra)) + elif change == 'dtype': + model.float() + elif change == 'adapter': + adapter.weights.add_(.1) + elif change == 'observed': + adapter.observed_mix = 1. + elif change == 'context': + engine.scorer.explicit_context = QueryContext([(0, 0, 2)], 4, 2) + elif change == 'precision': + monkeypatch.setattr('dicee.models._inference.float32_precision_token', lambda: ('changed',)) + actual = engine.predict(query) + assert engine.last_info['raw_rows'] == 1 + expected = QueryAnswerer(model, adapter=adapter, context=engine.scorer.context, cache_bytes=0).predict(query) + torch.testing.assert_close(actual, expected) + + +def test_cached_adapter_verification_still_rejects_mutations(raw, monkeypatch): + import dicee.query_answering.score_adapter as module + model = TableModel(raw).eval() + digest = state_fingerprint(model) + adapter = QueryScoreAdapter('global', metadata={'backbone_state_sha256': digest}) + real_hash, calls = module.state_fingerprint, [] + def counted(value): + calls.append(value) + return real_hash(value) + monkeypatch.setattr(module, 'state_fingerprint', counted) + engine = QueryAnswerer(model, adapter=adapter) + engine.predict((0, (0,))) + engine.predict((0, (0,))) + assert len(calls) == 1 + with torch.no_grad(): + model.table.add_(1) + with pytest.raises(ValueError, match='modified backbone'): + engine.predict((0, (0,))) + assert len(calls) == 2 + + +def test_inference_tensors_and_training_callers_do_not_persist_rows(raw): + for inference in (False, True): + with torch.inference_mode(inference): + model = TableModel(raw) + if inference: + model.eval() + engine = QueryAnswerer(model) + engine.predict((0, (0,))) + engine.predict((0, (0,))) + assert engine.last_info['raw_rows'] == 1 + + +def test_graph_context_validation_uses_identity_and_detects_changes(monkeypatch): + from dicee.models import ULTRA + model = ULTRA(dict(num_entities=3, num_relations=2, ultra_dim=8, ultra_num_layers=2)) + model.set_graph([(0, 0, 1)], inverse_relations={0: 1}).eval() + context = QueryContext.from_model(model) + def no_deep_comparison(*args): + raise AssertionError('must not walk the triples for equality') + monkeypatch.setattr(QueryContext, '__eq__', no_deep_comparison) + engine = QueryAnswerer(model, context=context) + engine.predict((0, (0,))) + engine.predict((0, (0,))) + assert engine.last_info['raw_rows'] == 0 + model.set_graph([(0, 0, 2)], inverse_relations={0: 1}) + with pytest.raises(ValueError, match='must match'): + engine.predict((0, (0,))) + + +def test_flock_persistent_cache_tracks_sampling(): + from dicee.models import Flock + model = Flock(dict(num_entities=4, num_relations=2, flock_dim=8, flock_walk_num=2, + flock_walk_len=8, flock_refinements=2)).set_graph([(0, 0, 1), (1, 1, 2)]).eval() + engine = QueryAnswerer(model, seed=13) + query = (0, (0,)) + engine.predict(query) + engine.predict(query) + assert engine.last_info['raw_rows'] == 0 + for attr, value in [('seed', 14), ('samples', 2)]: + setattr(engine.scorer, attr, value) + actual = engine.predict(query) + assert engine.last_info['raw_rows'] == 1 + expected = QueryAnswerer(model, seed=engine.scorer.seed, samples=engine.scorer.samples, cache_bytes=0).predict(query) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA unavailable') +def test_adapter_device_copy_refresh_and_training_gradients(raw): + adapter = QueryScoreAdapter('global') + logits = torch.tensor(raw[0, 0][None], device='cuda', dtype=torch.float64) + with torch.no_grad(): + before = adapter(logits) + adapter.weights[1, 0].add_(.2) + after = adapter(logits) + assert (after > before).all() + torch.testing.assert_close(after, QueryScoreAdapter('global', weights=adapter.weights).cuda()(logits)) + adapter(logits).sum().backward() + assert adapter.weights.grad is not None and torch.isfinite(adapter.weights.grad).all() + + +def test_eval_failure_restores_modes_and_does_not_poison_cache(raw, monkeypatch): + model = TableModel(raw).eval() + engine = QueryAnswerer(model) + original = model.forward_k_vs_all + def fail(_): + raise RuntimeError('scoring failed') + monkeypatch.setattr(model, 'forward_k_vs_all', fail) + with pytest.raises(RuntimeError, match='scoring failed'): + engine.predict((0, (0,))) + assert not model.training and not engine.scorer._evaluating + monkeypatch.setattr(model, 'forward_k_vs_all', original) + actual = engine.predict((0, (0,))) + torch.testing.assert_close(actual, model.table[0, 0].sigmoid()) + + +def test_cache_separates_raw_logits_and_bypasses_autocast(raw): + model = TableModel(raw).eval() + engine = QueryAnswerer(model) + query = (0, (0,)) + engine.predict(query) + logits = engine.predict(query, use_logits=True) + torch.testing.assert_close(logits, model.table[0, 0]) + assert engine.last_info['raw_rows'] == 1 + scores = engine.predict(query) + torch.testing.assert_close(scores, model.table[0, 0].sigmoid()) + assert engine.last_info['raw_rows'] == 1 + with torch.autocast('cpu', dtype=torch.bfloat16): + for _ in range(2): + engine.predict(query) + assert engine.last_info['raw_rows'] == 1 + + +def test_membership_threshold_zeroes_low_memberships_like_ultraquery_lp(tmp_path): + model = TableModel(np.zeros((4, 1, 4))) + adapter = QueryScoreAdapter('global', 1., membership_threshold=.5, metadata={'backbone_state_sha256': state_fingerprint(model)}) + raw = torch.tensor([[-2., 0., .1, 3.]], dtype=torch.float64) + observed, base = QueryContext([(0, 0, 0)], 4, 1).features([(0, 0)], device='cpu') + scores = adapter(raw, observed, base) + # Memberships .12, .5, .52, .95: those at or below .5 become zero, unless observed (entity 0 is an observed fact). + assert scores[0, 0] == 0 and scores[0, 1] == -torch.inf + torch.testing.assert_close(scores[0, 2:], torch.nn.functional.logsigmoid(raw[0, 2:]), rtol=0, atol=0) + adapter.save(tmp_path / 'adapter.json') + loaded = QueryScoreAdapter.load(tmp_path / 'adapter.json', model=model) + assert loaded.configuration == adapter.configuration and loaded.configuration['membership_threshold'] == .5 + # Adapters without a threshold keep their configuration, so existing artifacts and cache keys do not change. + assert 'membership_threshold' not in QueryScoreAdapter('global').configuration + with pytest.raises(ValueError, match='membership_threshold'): + QueryScoreAdapter('global', membership_threshold=1.) + + +def test_membership_threshold_runs_in_the_executor_and_keys_its_cache(): + adapter = QueryScoreAdapter('global') + engine = QueryAnswerer(TableModel(np.random.default_rng(3).normal(size=(5, 2, 5))).eval(), adapter=adapter) + before = engine.predict(((0, (0,)), (1, (1, -2)))) + adapter.membership_threshold = .6 + after = engine.predict(((0, (0,)), (1, (1, -2)))) + # Zero memberships are ordinary values for the executor, also under negation. + assert torch.isfinite(after).all() and (after == 0).any() and (after > 0).any() and not torch.equal(before, after) + + +def test_training_free_calibrations_follow_cqd_hybrid_and_qto(tmp_path): + model = TableModel(np.zeros((4, 1, 4))) + raw = torch.tensor([[-2., 0., 1., 3.], [5., 5., 5., 5.]], dtype=torch.float64) + observed, base = QueryContext([(0, 0, 0), (0, 0, 1)], 4, 1).features([(0, 0), (1, 0)], device='cpu') + minmax = QueryScoreAdapter('global', 0., fixed_calibration='minmax')(raw, observed, base).exp() + # CQD-Hybrid: 0.9 (s - min) / (max - min); a constant row has no order and gets zero everywhere. + torch.testing.assert_close(minmax[0], .9 * (raw[0] + 2) / 5, rtol=0, atol=1e-15) + assert bool((minmax[1] == 0).all()) + softmax = QueryScoreAdapter('global', 0., fixed_calibration='softmax-degree')(raw, observed, base).exp() + # QTO: softmax times the observed tail count (two for (0, 0), at least one otherwise), capped at 0.9999. + expected = (raw.softmax(1) * torch.tensor([[2.], [1.]], dtype=torch.float64)).clamp_max(.9999) + torch.testing.assert_close(softmax, expected, rtol=1e-12, atol=0) + facts = QueryScoreAdapter('global', 1., fixed_calibration='softmax-degree', + metadata={'backbone_state_sha256': state_fingerprint(model), 'calibration': 'documentation only'}) + scores = facts(raw, observed, base) + assert scores[0, 0] == 0 and scores[0, 1] == 0 and scores[0, 3] < 0 + facts.save(tmp_path / 'adapter.json') + loaded = QueryScoreAdapter.load(tmp_path / 'adapter.json', model=model) + assert loaded.configuration == facts.configuration and loaded.fixed_calibration == 'softmax-degree' and loaded.needs_context + assert loaded.metadata['calibration'] == 'documentation only' + # The default sigmoid keeps every existing configuration, artifact and cache key unchanged. + assert 'fixed_calibration' not in QueryScoreAdapter('global').configuration and not QueryScoreAdapter('global').needs_context + with pytest.raises(ValueError, match='Training-free'): + QueryScoreAdapter('global', fixed_calibration='minmax', weights=[[1.], [0.]]) + with pytest.raises(ValueError, match='Training-free'): + QueryScoreAdapter('context_scores', fixed_calibration='softmax-degree') + with pytest.raises(ValueError, match='fixed calibration'): + QueryScoreAdapter('global', fixed_calibration='temperature') + with pytest.raises(ValueError, match='context graph'): + QueryScoreAdapter('global', fixed_calibration='softmax-degree')(raw) + + +def test_observed_ties_follow_the_backbone_only_when_asked(): + raw = torch.tensor([[1., 3., 2., 0.]], dtype=torch.float64) + observed, base = QueryContext([(0, 0, 0), (0, 0, 1), (0, 0, 2)], 4, 1).features([(0, 0)], device='cpu') + adapter = QueryScoreAdapter('global', 1.) + plain = adapter(raw, observed, base) + # Default: every known tail is exactly one, so a beam keeps the lowest entity IDs among them. + assert bool((plain[0, :3] == 0).all()) and plain[0, 3] < 0 + adapter.observed_tie_break = True + ordered = adapter(raw, observed, base) + # Known tails stay within a billionth of one, ordered by the backbone (entity 1 > 2 > 0); other entities are unchanged. + assert bool((ordered[0, :3] <= 0).all()) and bool((ordered[0, :3] > -1e-9).all()) + assert ordered[0, 1] > ordered[0, 2] > ordered[0, 0] + assert ordered[0, 3] == plain[0, 3] + assert 'observed_tie_break' not in adapter.configuration + + +def test_softmax_calibration_variants_and_known_tail_masking(): + raw = torch.tensor([[0., 30., 30.5, 2., 1.], [1., 2., 3., 4., 5.]], dtype=torch.float64) + observed, base = QueryContext([(0, 0, 0), (0, 0, 3), (0, 0, 4)], 5, 1).features([(0, 0), (1, 0)], device='cpu') + softmax = QueryScoreAdapter('global', 0., fixed_calibration='softmax')(raw, observed, base).exp() + # d = 1: the plain softmax, capped below one. + torch.testing.assert_close(softmax, raw.softmax(1).clamp_max(.9999), rtol=1e-12, atol=0) + qto = QueryScoreAdapter('global', 0., fixed_calibration='softmax-degree')(raw, observed, base) + ties = QueryScoreAdapter('global', 0., fixed_calibration='softmax-degree-ties')(raw, observed, base) + # QTO's memberships up to a millionth; with d = 3, entities 1 and 2 of row 0 both exceed the cap and tie under QTO. + assert float((qto - ties).abs().max()) <= 1e-6 + assert qto[0, 1] == qto[0, 2] and ties[0, 2] > ties[0, 1] + assert bool((qto[1] == ties[1]).all()) + masked = QueryScoreAdapter('global', 1., fixed_calibration='softmax-degree', mask_known_logits=True) + scores = masked(raw, observed, base) + # Known tails (entities 0, 3 and 4 of row 0) leave the softmax and keep membership one through the override. + expected = (raw[0, 1:3].softmax(0) * 3).clamp_max(.9999).log() + torch.testing.assert_close(scores[0, 1:3], expected, rtol=1e-12, atol=0) + assert scores[0, 0] == 0 and scores[0, 3] == 0 and scores[0, 4] == 0 + assert masked.configuration['mask_known_logits'] is True and 'mask_known_logits' not in QueryScoreAdapter('global').configuration + with pytest.raises(ValueError, match='Masking known-tail'): + QueryScoreAdapter('global', fixed_calibration='minmax', mask_known_logits=True) + # FIT: the softmax times the observed tail count over the softmax mass on the observed tails, so that the observed + # tails of row 0 (entities 0, 3 and 4) together hold three before the cap; row 1 has none and keeps the softmax. + fit = QueryScoreAdapter('global', 0., fixed_calibration='softmax-mass')(raw, observed, base).exp() + share = raw.softmax(1) + factor = 3 / share[0, [0, 3, 4]].sum() + torch.testing.assert_close(fit[0], (share[0] * factor).clamp_max(.9999), rtol=1e-12, atol=0) + torch.testing.assert_close(fit[1], share[1].clamp_max(.9999), rtol=1e-12, atol=0) + # Before the cap the observed tails hold exactly their count. Here the two unobserved entities hold nearly all of the + # softmax mass, so the factor is large and lifts them to the cap, above two of the three observed tails. + torch.testing.assert_close((share[0, [0, 3, 4]] * factor).sum(), torch.tensor(3., dtype=torch.float64), rtol=1e-12, atol=0) + assert float(factor) > 1e11 and fit[0, 1] == fit[0, 2] == .9999 and fit[0, 0] < fit[0, 4] < .9999 + with pytest.raises(ValueError, match='softmax-mass'): + QueryScoreAdapter('global', fixed_calibration='softmax-mass', mask_known_logits=True) diff --git a/tests/test_query_generator.py b/tests/test_query_generator.py index a6c071b83..c81037516 100644 --- a/tests/test_query_generator.py +++ b/tests/test_query_generator.py @@ -58,16 +58,29 @@ def test_save_queries_rejects_unknown_type(generator, tmp_path): generator.save_queries("not-a-type", 5, str(tmp_path)) +class _LogitTable(torch.nn.Module): + """Atomic model whose raw logits depend on the relation only, the same for every head.""" + + name = "LogitTable" + + def __init__(self, logits): + super().__init__() + num_relations, num_entities = logits.shape + self.table = torch.nn.Parameter(logits.to(torch.float64).expand(num_entities, num_relations, num_entities).clone()) + self.num_entities, self.num_relations = num_entities, num_relations + + def forward_k_vs_all(self, pairs): + return self.table[pairs[:, 0], pairs[:, 1]] + + class _StubKGE(KGE): """KGE whose atomic predictions are fixed raw logits, so no training is needed.""" def __init__(self): self.entity_to_idx = {f"e{i}": i for i in range(3)} - self._logits = {"a": torch.tensor([-3.0, -1.0, 2.0]), "b": torch.tensor([-2.0, -4.0, 3.0])} - - def predict(self, h=None, r=None, t=None, within=None, logits=True): - scores = self._logits[r[0] if isinstance(r, list) else r] - return scores if logits else torch.sigmoid(scores) + self.relation_to_idx = {"a": 0, "b": 1} + # answer_multi_hop_query scores atomic queries with the model's forward_k_vs_all. + self.model = _LogitTable(torch.tensor([[-3.0, -1.0, 2.0], [-2.0, -4.0, 3.0]])) def test_intersection_defaults_to_probabilities_and_ranks_correctly(): diff --git a/tests/test_read_from_disk_splits.py b/tests/test_read_from_disk_splits.py new file mode 100644 index 000000000..887e8dc37 --- /dev/null +++ b/tests/test_read_from_disk_splits.py @@ -0,0 +1,38 @@ +"""ReadFromDisk routes the split files of a dataset folder by their file names.""" + +from types import SimpleNamespace + +import pytest + +from dicee.read_preprocess_save_load_kg.read_from_disk import ReadFromDisk + +TRAIN = [('a', 'r', 'b'), ('b', 'r', 'c')] +VALID = [('a', 'r', 'c')] +TEST = [('c', 'r', 'a')] + + +def _write(path, triples) -> None: + path.write_text(''.join(f'{head}\t{relation}\t{tail}\n' for head, relation, tail in triples)) + + +def _rows(frame) -> list: + return [tuple(row) for row in frame.values.tolist()] + + +@pytest.mark.parametrize('folder', ['train-valid-test-kg', 'test', 'valid']) +def test_split_files_are_routed_by_file_name(tmp_path, folder): + """A folder name that contains a split word must not decide the split of its files.""" + dataset = tmp_path / folder + dataset.mkdir() + _write(dataset / 'train.txt', TRAIN) + _write(dataset / 'valid.txt', VALID) + _write(dataset / 'test.txt', TEST) + kg = SimpleNamespace(path_single_kg=None, sparql_endpoint=None, dataset_dir=str(dataset), read_only_few=None, + sample_triples_ratio=None, backend='pandas', separator=r'\s+', add_noise_rate=None, + eval_model='train_val_test') + + ReadFromDisk(kg).start() + + assert _rows(kg.raw_train_set) == TRAIN + assert _rows(kg.raw_valid_set) == VALID + assert _rows(kg.raw_test_set) == TEST diff --git a/tests/test_unit_kge_inference.py b/tests/test_unit_kge_inference.py index 626096710..f96b6c0cb 100644 --- a/tests/test_unit_kge_inference.py +++ b/tests/test_unit_kge_inference.py @@ -12,6 +12,8 @@ import pytest import torch +from dicee.knowledge_graph_embeddings import KGE + # --------------------------------------------------------------------------- # Fixtures: Mock Model and KGE Instance # --------------------------------------------------------------------------- @@ -53,13 +55,8 @@ def mock_forward(x): @pytest.fixture def mock_kge_instance(tmp_path): - """Create a minimal real KGE-like instance for validation testing. - - This uses a real object (not MagicMock) so that real validation - logic in methods like predict_topk, predict, etc. actually executes. - """ - # Create a simple object that has the required attributes - class MinimalKGE: + """Use real KGE methods with a mock model, without loading a checkpoint.""" + class MinimalKGE(KGE): def __init__(self): self.entity_to_idx = {f"e{i}": i for i in range(100)} self.idx_to_entity = {i: f"e{i}" for i in range(100)} @@ -79,17 +76,7 @@ def __init__(self): self.model.to = MagicMock(return_value=self.model) self.model.entity_embeddings = MagicMock(return_value=torch.randn(100, 64)) - kge = MinimalKGE() - - # Attach real validation methods from KGE - from dicee.knowledge_graph_embeddings import KGE as RealKGE - kge.to = RealKGE.to.__get__(kge, type(kge)) - kge.predict_topk = RealKGE.predict_topk.__get__(kge, type(kge)) - kge.predict = RealKGE.predict.__get__(kge, type(kge)) - kge.get_transductive_entity_embeddings = RealKGE.get_transductive_entity_embeddings.__get__(kge, type(kge)) - kge.__str__ = RealKGE.__str__.__get__(kge, type(kge)) - - return kge + return MinimalKGE() # --------------------------------------------------------------------------- @@ -110,6 +97,19 @@ def test_to_device_cuda_valid(self, mock_kge_instance): mock_kge_instance.to("cuda") mock_kge_instance.model.to.assert_called_with("cuda") + @pytest.mark.parametrize("device", ["cpu", "cuda"]) + def test_to_clears_query_cache_before_transfer(self, mock_kge_instance, device): + mock_kge_instance._query_engine = object() + mock_kge_instance._query_engine_key = object() + + def transfer(target): + assert mock_kge_instance._query_engine is None + assert mock_kge_instance._query_engine_key is None + + mock_kge_instance.model.to.side_effect = transfer + mock_kge_instance.to(device) + mock_kge_instance.model.to.assert_called_once_with(device) + def test_to_device_invalid_raises_valueerror(self, mock_kge_instance): """to() with invalid device should raise ValueError (not AssertionError).""" with pytest.raises(ValueError, match="Device must be either cpu or cuda"):