diff --git a/LLM_GUIDANCE.md b/LLM_GUIDANCE.md index a630c36..9975c34 100644 --- a/LLM_GUIDANCE.md +++ b/LLM_GUIDANCE.md @@ -62,17 +62,21 @@ Naming a real ID that you have not seen in a tool result this conversation count ## Search Filter Cookbook -Unfiltered `search_terms` calls return deprecated terms, scRNAseq artifacts, and unrelated entity types mixed in with what the user wants. Use `filter_types` from the start. Common recipes: - -| User asks about | `filter_types` | `exclude_types` | -|---|---|---| -| Neuron types/classes | `["neuron", "class"]` | `["deprecated"]` | -| Individual neurons (with images) | `["neuron", "has_image"]` | `["deprecated"]` | -| Neurons with connectome data | `["neuron", "has_neuron_connectivity"]` | `["deprecated"]` | -| Brain regions / neuropils | `["anatomy"]` | `["deprecated"]` | -| Genes | `["gene"]` | `["deprecated"]` | -| Expression patterns / driver lines | `["expression_pattern"]` | `["deprecated"]` | -| Datasets | `["dataset"]` | — | +`search_terms` runs the search virtualflybrain.org itself runs — the same Solr query, the same filters and boosts, the same final sort — so what you get back is what a user would see on the site. Deprecated terms are already excluded server-side, so you do not need to ask for that. + +Unfiltered calls still mix scRNAseq artifacts and unrelated entity types in with what the user wants. Use `filter_types` from the start. Common recipes: + +| User asks about | `filter_types` | +|---|---| +| Neuron types/classes | `["neuron", "class"]` | +| Individual neurons (with images) | `["neuron", "has_image"]` | +| Neurons with connectome data | `["neuron", "has_neuron_connectivity"]` | +| Brain regions / neuropils | `["anatomy"]` | +| Genes | `["gene"]` | +| Expression patterns / driver lines | `["expression_pattern"]` | +| Datasets | `["dataset"]` | + +**Do not guess type names.** There are over 200 of them and they change as data is added. Call `list_search_facets` — optionally with `contains` to narrow, e.g. `contains: "neuron"` — and use what it returns. An unknown name is rejected outright, with suggestions. **Stage filtering — only when the user is specific.** VFB covers adult, larval, and embryonic data, and many anatomical FBbt terms are stage-agnostic (the "antennal lobe" class covers all life stages). Do NOT add `"adult"` by default — you will hide the generic class and any larval/embryonic results. @@ -83,8 +87,12 @@ Unfiltered `search_terms` calls return deprecated terms, scRNAseq artifacts, and Other useful options on `search_terms`: - `boost_types: ["has_image", "has_neuron_connectivity"]` — soft-ranks the most data-rich entities first without excluding others. +- `demote_types: [...]` — the mirror image: sinks matches to the bottom of the ranking without excluding them. Prefer this to `exclude_types` when you suspect the user might still want the demoted results, just not first. A type listed in both `boost_types` and `demote_types` is treated as boosted. +- `unique: false` — one row per matching *synonym* instead of one per term. Use it when the useful question is "which name matched?", and expect the same ID to repeat. The default (`true`) gives one row per term. - `minimize_results: true` — limits to top 10 and adds truncation metadata. Use for exploratory searches to avoid filling context with irrelevant matches. -- `auto_fetch_term_info: true` — when an exact label match is found, folds `get_term_info` into the same response, saving a round trip. +- `auto_fetch_term_info: true` — when the query matches one term's name exactly, folds `get_term_info` into the same response, saving a round trip. + +**Reading the counts.** The response reports them separately and they mean different things: `returned` is how many rows you were given, `total` is the length of the ranked list they came from, `distinct_terms` is how many distinct terms matched, and `solr_matches` is how many terms Solr matched before ranking. Under `unique: false` the rows are synonyms, so `total` counts names and can exceed `solr_matches`, which counts terms — the `_note` spells that out when it happens. If `solr_matches` is much larger than `candidate_pool`, only the top slice was ranked. Never report `returned` as if it were the number of matches. --- @@ -181,17 +189,19 @@ Returns tabular data from pre-computed analyses: ### **Search Results Response (`search_terms`)** -Returns entity search results from SOLR. Supports optional type-based filtering and result control: +Runs the website's own search. Parameters: - **`filter_types`**: Hard include — results must have ALL specified `facets_annotation` values (AND logic) - **`exclude_types`**: Hard exclude — results must NOT have any of these types - **`boost_types`**: Soft boost — results with these types rank higher without excluding others -- **`start`**: Pagination start index (default 0) -- **`rows`**: Number of results to return (default 150, max 1000) -- **`minimize_results`**: When true, limits results and adds truncation metadata for reduced context -- **`auto_fetch_term_info`**: When true and exact match found, includes term info in response +- **`demote_types`**: Soft demote — results with these types rank lower without being excluded. A type in both `boost_types` and `demote_types` is boosted +- **`unique`**: One row per term (default `true`). `false` gives one row per matching synonym +- **`start`**: Page start index (default 0) +- **`rows`**: Rows to return (default 150, max 1000) +- **`minimize_results`**: Return only the top 10, with reduced fields, for a first look +- **`auto_fetch_term_info`**: When the query matches one term's name exactly, include that term's info -Available filter types are loaded dynamically from Solr at server startup, so the tool description always lists current values. +Call `list_search_facets` for valid type names — do not guess them, and do not rely on any list embedded in documentation, which will drift. **Basic search:** ```json @@ -200,22 +210,27 @@ Available filter types are loaded dynamically from Solr at server startup, so th } ``` -**Filtered search (only adult neurons with images):** +**Filtered search (adult neurons with images):** ```json { "query": "medulla", - "filter_types": ["neuron", "adult", "has_image"], - "exclude_types": ["deprecated"] + "filter_types": ["neuron", "adult", "has_image"] } ``` -**Minimized search with pagination:** +**One row per synonym, to see which name matched:** +```json +{ + "query": "Kenyon cell", + "unique": false +} +``` + +**Minimized first look, then paging:** ```json { "query": "medulla", - "minimize_results": true, - "start": 0, - "rows": 20 + "minimize_results": true } ``` @@ -230,41 +245,51 @@ Available filter types are loaded dynamically from Solr at server startup, so th **Response:** ```json { - "response": { - "numFound": 1234, - "docs": [ - { - "short_form": "FBbt_00007484", - "label": "antennal lobe", - "synonym": ["antennal lobe"], - "id": "http://purl.obolibrary.org/obo/FBbt_00007484", - "facets_annotation": ["Adult", "Nervous_system"], - "unique_facets": ["adult antennal lobe", "nervous system"] - } - ], - "_truncation": { - "truncated": true, - "shown": 10, - "totalAvailable": 1234, - "canRequestMore": true + "query": "antennal lobe", + "unique": true, + "start": 0, + "returned": 3, + "total": 438, + "distinct_terms": 438, + "solr_matches": 438, + "candidate_pool": 500, + "_note": "Showing results 0-3 of 438. To see more, re-run with start=3 (same rows).", + "results": [ + { + "label": "antennal lobe cPIN (antennal lobe commissural pioneer interneuron)", + "original_label": "antennal lobe commissural pioneer interneuron", + "short_form": "FBbt_00052563", + "id": "http://purl.obolibrary.org/obo/FBbt_00052563", + "facets_annotation": ["Entity", "Class", "Neuron", "Anatomy", "Cell", "Nervous_system", "has_subClass"], + "unique_facets": ["Nervous_system", "Neuron"] } - }, - "_term_info": { - "Id": "FBbt_00007484", - "Name": "antennal lobe", - "Types": ["Class"], - "Definition": "The antennal lobe..." - } + ], + "term_info": { "Id": "FBbt_00003924", "Name": "antennal lobe" } } ``` **Key Fields:** -- **short_form**: VFB/FlyBase identifier -- **label**: Primary display name -- **facets_annotation**: Categorization tags (also used for filtering) -- **id**: Full ontology IRI -- **_truncation**: Metadata when `minimize_results=true` indicating if results were limited -- **_term_info**: Automatically fetched term details when `auto_fetch_term_info=true` and exact match found +- **short_form**: VFB/FlyBase identifier — use this for `get_term_info`, `run_query`, and URLs +- **label**: The site's display form, which is not the plain name. It is either `"name (ID)"` or `"matched synonym (name)"`, so it tells you *why* the row matched +- **original_label**: The plain name. Compare against this, not `label`, when checking whether a result is what the user asked for +- **facets_annotation**: Categorisation tags, and the vocabulary the four type filters draw on +- **returned / total / distinct_terms / solr_matches**: Four different numbers — rows given to you, length of the ranked list, distinct terms matched, and terms Solr matched before ranking. Report the right one; `returned` is never the answer to "how many are there?" +- **candidate_pool**: How many documents Solr was asked for before ranking. If `solr_matches` exceeds it, the list is not exhaustive — narrow the query or add `filter_types` rather than paging to the end +- **_note**: Says what was truncated, collapsed, or approximated. Read it before summarising +- **term_info**: Present when `auto_fetch_term_info` was set and the query matched one term's name exactly + +### **Facet Vocabulary Response (`list_search_facets`)** + +```json +{ + "count": 12, + "total": 233, + "contains": "neuron", + "facets": [{ "name": "Neuron", "docs": 48213 }] +} +``` + +`docs` is how many documents carry that facet, which is a useful sanity check: a facet with a handful of documents will not usefully narrow a search. If the response carries a `source` of `"static snapshot bundled with this MCP server"`, the live vocabulary was unreachable and you are seeing a stale list — a name missing from it may still be valid. ## FlyBase Entity Resolution & Stocks Workflow @@ -478,6 +503,12 @@ run_query(id="VFB_00104glj", query_type="NeuronInputsTo") 5. **Execute** — Call `query_connectivity` with confirmed parameters. +**Reading a paged connectivity result.** `query_connectivity` returns a strongest-first page of `limit` rows (default 50) plus a `summary` computed over **every** connection found, not just the page. Answer from the summary; quote a handful of rows as illustration. A single class at `weight=5` can find over 50,000 connections, so the page you see is often a tiny fraction — `count` is the real total, `returned` is what you were given, and the `_note` says so explicitly. Only page with `offset` if the user asks for specific further rows; if they want the shape of the whole result set, `group_by_class=true` is the better move. + +The summary contains: total `connections`, the weight column's `min`/`max`/`total`/`mean`, `by_dataset` counts, `distinct_class_pairs`, `distinct_upstream_neurons`, `distinct_downstream_neurons`, and `top_class_pairs` ranked by connection count. That is usually a better answer to "how do these two classes connect?" than any 50 rows would be. + +Note that a class label may be several names joined with `|` — a neuron that is both `adult GABAergic neuron` and `proximal medullary amacrine neuron Pm2` appears as `"adult GABAergic neuron|proximal medullary amacrine neuron Pm2"`. That is one class set, not two classes; when presenting it to a user, the most specific name in the set is usually the one they want. + **Performance rules for `query_connectivity`:** - Always start with the default `weight = 5`. There is no universal "good" weight — it varies by cell type. - Single-end queries (only upstream or only downstream set) are **slower** than both-ends queries because they return more results. If the user only cares about one direction, prefer `DownstreamClassConnectivity` or `UpstreamClassConnectivity` via `run_query` instead — they are pre-indexed and fast. @@ -496,16 +527,18 @@ Or for `query_connectivity`: ``` Query: - Upstream type: transmedullary neuron Tm1 (FBbt_00003789) -- Downstream type: T3 neuron (FBbt_00047727) +- Downstream type: T3 neuron (FBbt_00003730) - Min. weight: 5 -- Excluded DBs: hb, fafb -Results: 142 connections across 28 upstream neurons → 85 downstream neurons +- Excluded DBs: mc +Results: 7309 connections across 28 upstream neurons → 85 downstream neurons ``` **Result formatting:** - **≤50 rows:** Show full table. - **>50 rows:** Show top 20 sorted by weight descending. Include summary stats (total connections, unique partners, weight range). Note that results are truncated. +For `query_connectivity` the summary stats are already computed for you over the full result set — use those figures rather than deriving them from the rows in front of you, which are only a page. + **Column guide by query type:** | Query type | Key columns | @@ -705,8 +738,9 @@ A term is a template brain if its `SuperTypes` array from `get_term_info` includ ### **1. Start with Search** - Use `search_terms` to find relevant entities - Use `filter_types` to narrow results by entity type (e.g., `["neuron"]`, `["gene"]`, `["expression_pattern"]`) -- Use `exclude_types` to remove unwanted results (e.g., `["deprecated"]`) -- Use `boost_types` to prioritize results with useful data (e.g., `["has_image", "has_neuron_connectivity"]`) +- Use `list_search_facets` to find valid type names instead of guessing them +- Use `exclude_types` to remove unwanted result types (deprecated terms are already excluded server-side) +- Use `boost_types` to prioritize results with useful data (e.g., `["has_image", "has_neuron_connectivity"]`), and `demote_types` to push a type down without hiding it - For large result sets, use `minimize_results: true` to limit to top 10 and reduce context usage - For exact term matches, use `auto_fetch_term_info: true` to get immediate detailed information - Use `start` and `rows` for pagination when exploring large result sets @@ -774,13 +808,14 @@ A term is a template brain if its `SuperTypes` array from `get_term_info` includ - Cell type hierarchy: Search with `filter_types: ["neuron", "class"]` → `get_hierarchy` with `relationship: "subclass_of"` - Datasets: Search with `filter_types: ["dataset"]` to find available datasets - Exact term lookup: Use `auto_fetch_term_info: true` for immediate detailed information on exact matches -- Exclude noise: Always consider `exclude_types: ["deprecated"]` to remove obsolete entities +- Facet names: Call `list_search_facets` rather than guessing type names — there are over 200 and they change ### **Error Handling** See also: Rules 2 and 3 at the top of this document. -- If `search_terms` returns no good matches, try alternative spellings, synonyms, broader terms, or different `filter_types` from the cookbook. +- If `search_terms` returns no good matches, try alternative spellings, synonyms, broader terms, or different `filter_types` from the cookbook. Setting `unique: false` can help here — it shows which synonym matched, which often explains a surprising result. +- If `search_terms` or `query_connectivity` returns a rejection message, read it: an unknown facet name comes back with suggested alternatives, and a connectivity rejection names the missing or unresolvable parameter. Fix the call from the message rather than retrying it unchanged. - If `run_query` fails or returns empty, call `get_term_info` on the ID and pick a different `query_type` from the `Queries` array. The error message will list the valid query_types — use them. - If no MCP call answers the question, tell the user clearly what you tried and what you found. Do **not** fall back to training-data answers (no fabricated FBbt/FBgn/FBst IDs, driver names, citations, or numbers). - Network timeouts: suggest retrying. `query_connectivity` is the only slow tool — others should respond in seconds. diff --git a/README.md b/README.md index 1a2363f..dfe749b 100644 --- a/README.md +++ b/README.md @@ -150,11 +150,12 @@ The MCP server exposes the following tools (available to assistants like Claude - `get_term_info` — Get detailed metadata for a VFB ID - `run_query` — Run a precomputed analysis query for a VFB ID (see the `Queries` field from `get_term_info`) -- `search_terms` — Search VFB entities by text with filtering / boosting options +- `search_terms` — Search VFB entities by text with filtering / boosting options. This is the same search virtualflybrain.org itself runs +- `list_search_facets` — List the `facets_annotation` type names that `search_terms`' `filter_types` / `exclude_types` / `boost_types` / `demote_types` accept, optionally filtered by substring - `resolve_entity` — Resolve an unresolved FlyBase-related query string (e.g., `P{VT054895-GAL4.DBD}` or a driver line / cell type label) to VFB/FlyBase IDs and metadata (not the same as VFB term search) - `resolve_combination` — Resolve an unresolved split-GAL4 combination name or synonym into its component IDs - `list_connectome_datasets` — List available connectome datasets (e.g., Hemibrain, FAFB) -- `query_connectivity` — Query connectivity across connectome datasets using upstream/downstream filters +- `query_connectivity` — Query connectivity across connectome datasets using upstream/downstream filters, returned as a strongest-first page plus a summary computed over every connection found - `get_hierarchy` — Traverse the ontology hierarchy for a VFB ID: `part_of` (region/tissue structure) and/or `subclass_of` (cell-type taxonomy), ancestors and/or descendants ## 🛠️ Local Installation @@ -288,17 +289,40 @@ Execute predefined queries on VFB data. FlyBase stocks and split-GAL4 combination publications are run_query query_types too: `FindStocks` and `FindComboPublications`. ### search_terms -Search for VFB terms using the Solr search server with optional filtering and result control. +Search for VFB terms. This calls VFBquery's `/search`, which is the search virtualflybrain.org itself runs — the same Solr query, the same filters and boosts, the same final sort — so a result here is the result a user would see on the site. Deprecated terms are already excluded server-side; there is no need to ask for that. **Parameters:** - `query` (string): Search query (e.g., "medulla") -- `filter_types` (array, optional): Filter results to only include items matching ALL of these facets_annotation types (AND logic) -- `exclude_types` (array, optional): Exclude results matching ANY of these facets_annotation types (OR logic) -- `boost_types` (array, optional): Boost ranking of results matching these facets_annotation types without excluding others -- `start` (number, optional): Pagination start index (default 0) - use to get results beyond the first page -- `rows` (number, optional): Number of results to return (default 150, max 1000) - use smaller numbers for focused searches -- `minimize_results` (boolean, optional): When true, limit results to top 10 for initial searches and add truncation metadata (default false) -- `auto_fetch_term_info` (boolean, optional): When true and an exact match is found, automatically fetch and include term info in the response (default false) +- `filter_types` (array, optional): Keep only results matching ALL of these facets_annotation types (AND logic) +- `exclude_types` (array, optional): Drop results matching ANY of these facets_annotation types (OR logic) +- `boost_types` (array, optional): Lift results matching these types up the ranking without excluding others +- `demote_types` (array, optional): Sink results matching these types to the bottom of the ranking without excluding them. Ignored for a type that also appears in `boost_types` +- `unique` (boolean, optional): One row per term (default true). Set false for one row per matching synonym, which shows *which* name matched at the cost of repeating IDs +- `start` (number, optional): Page start index (default 0) +- `rows` (number, optional): Rows to return (default 150, max 1000) +- `minimize_results` (boolean, optional): Return only the top 10 with reduced fields, for a first look (default false) +- `auto_fetch_term_info` (boolean, optional): When the query matches one term's name exactly, also fetch that term's info (default false) + +Type names come from the live vocabulary — there are over 200 of them and they change as data is added, so call `list_search_facets` rather than guessing. The response reports `returned` (rows given), `total` (length of the ranked list), `distinct_terms` and `solr_matches` (terms Solr matched before ranking) separately, so a truncated page never looks like a small result set. + +### list_search_facets +List the valid `facets_annotation` type names for the four type filters above, read from the live vocabulary. + +**Parameters:** +- `contains` (string, optional): Case- and separator-insensitive substring filter (e.g., "neuron", "nervous system") + +If the deployed VFBquery predates the `/facets` endpoint, this falls back to a snapshot bundled with the server and says so — names absent from a snapshot result may still be valid. + +### query_connectivity +Query synaptic connectivity between neuron classes across all connectome datasets. At least one of `upstream_type` or `downstream_type` is required. Results are ranked strongest-first and paged: you get `limit` rows plus a `summary` computed over **every** connection found — weight min/max/total/mean, per-dataset counts, distinct neuron counts, and the top class pairs — so the totals stay true even though the rows are truncated. A broad query can find tens of thousands of connections, which is why paging is on by default. + +**Parameters:** +- `upstream_type` / `downstream_type` (string, optional): Neuron class OWL ID or label. Anatomical regions are not accepted +- `weight` (number, optional): Minimum synapse count (recommended 5; use ≥50 when both ends are specified) +- `group_by_class` (boolean, optional): Aggregate to class pairs instead of neuron pairs — usually the better first call on a broad query +- `exclude_dbs` (array, optional): Dataset symbols to exclude (recommended `["hb","fafb"]`); see `list_connectome_datasets` +- `limit` (number, optional): Rows to return, strongest first (default 50; `0` for all) +- `offset` (number, optional): Row to start from within the ranking (default 0) ## 🧠 About VirtualFlyBrain diff --git a/TECHNICAL.md b/TECHNICAL.md index 8f514d3..14d5375 100644 --- a/TECHNICAL.md +++ b/TECHNICAL.md @@ -172,7 +172,16 @@ The server integrates with VirtualFlyBrain APIs: - **Term Info API**: `https://v3-cached.virtualflybrain.org/get_term_info` - **Query API**: `https://v3-cached.virtualflybrain.org/run_query` -- **Solr Search API**: `https://solr.virtualflybrain.org/solr/ontology/select` +- **Search API**: `https://v3-cached.virtualflybrain.org/search` +- **Facet vocabulary API**: `https://v3-cached.virtualflybrain.org/facets` +- **Connectivity API**: `https://v3-cached.virtualflybrain.org/query_connectivity` + +The server no longer talks to `solr.virtualflybrain.org` directly. It used to build +its own Solr query for search — its own `fq`, its own `bq`, its own `qf` — and that +construction was a copy of the website's, made once and then left to drift. Worse, +it skipped the website's refine/sort pass, so its ordering was never the website's +either. VFBquery's `/search` *is* the website's search, so search now goes through +it: one ranking, one place to fix it. ### MCP Tools Implementation @@ -187,14 +196,65 @@ The server integrates with VirtualFlyBrain APIs: - **API Call**: GET `run_query` with `offset`/`limit` #### search_terms -- **Input**: Search query with optional filters, pagination, and result control parameters -- **Output**: Search results with optional truncation metadata and auto-fetched term info -- **API Call**: GET to Solr search endpoint with enhanced parameter handling +- **Input**: Search query with optional `filter_types` / `exclude_types` / `boost_types` / `demote_types`, `unique`, `start`, `rows`, `minimize_results`, `auto_fetch_term_info` +- **Output**: `{query, unique, start, returned, total, distinct_terms?, solr_matches?, candidate_pool, _note?, results, term_info?}`. The counts are reported separately rather than collapsed into one number, so a truncated page cannot be mistaken for a small result set +- **API Call**: GET `/search` + +Two details of `/search` shape the implementation: + +- **`rows` on the wire is a candidate pool, not a page size.** It is how many documents + Solr is asked for before ranking, so it affects *which* results come back, not just + how many. The tool therefore sends `rows = min(1000, max(500, start + rows))` — an + ordinary request gets the website's own 500-candidate pool and identical ranking, and + only a caller paging past that widens the net. `/search` has no `offset`, so the page + slice is taken client-side. +- **`original_label` is the raw label; `label` is the refined display form** + (`"medulla (FBbt_00003748)"`, `"synonym (label)"`). Exact-match detection compares + `original_label` — comparing the display form, as the old code did, could never match. + +`unique` and `distinct_terms` are newer than some deployed VFBquery versions. Rather +than version-sniffing, the tool asks for `unique=true` and treats "did the response +echo `unique`?" as the capability probe, caching the answer for the process. Where the +server does not honour it, the tool refetches *without* `limit` and de-duplicates +locally — necessary because the rows past the limit are exactly the ones that survive +de-duplication — and says so in `_note`. + +#### list_search_facets +- **Input**: optional `contains` substring +- **Output**: `{count, total, contains?, source?, _note?, facets}` +- **API Call**: GET `/facets` + +On `404` (endpoint not deployed yet) or `503` (vocabulary unavailable) this falls back +to `STATIC_FACET_SNAPSHOT`, a snapshot of the names that used to be pasted into the +`search_terms` description. The snapshot is not maintained; the response labels its +source and warns that names missing from it may still be valid. + +#### query_connectivity +- **Input**: `upstream_type` / `downstream_type` (at least one required), `weight`, `group_by_class`, `exclude_dbs`, `limit`, `offset` +- **Output**: `{count, offset, limit, returned, ranked_by, summary, _note?, warnings?, resolved?, connections}` +- **API Call**: GET `/query_connectivity`, 5-minute timeout (live cross-dataset query, not cached) + +`/query_connectivity` has no paging of its own and returns every row it finds — a single +class at `weight=5` can be over 50,000 connections, which is not something to hand a +model whole. `limit`/`offset` are therefore applied client-side after ranking +strongest-first (stable, with the original index as tiebreak), and the `summary` is +computed over the **full** set so the totals remain true. + +One subtlety in the summary: class labels arrive from Neo4j as +`apoc.text.join(collect(distinct c.label),'|')`, and the order within that join is not +deterministic — the same logical pair of classes comes back as `"A|B"` on one row and +`"B|A"` on the next. Left alone that splits one class pair across several summary rows +and inflates `distinct_class_pairs`. The parts are sorted before keying, which gives one +stable spelling per label set. ### Error Handling - Axios HTTP client with timeout configuration -- Graceful fallback for API unavailability +- Graceful fallback for API unavailability, including capability probing rather than + version sniffing where a newer VFBquery adds a parameter or an endpoint +- Rejection bodies from `/search` and `/query_connectivity` are surfaced to the caller + rather than reduced to a status code — both endpoints explain what to change (an + unknown facet name comes back with suggestions), and that explanation is the useful part - Structured error responses following MCP protocol - Logging for debugging and monitoring diff --git a/examples.md b/examples.md index 2455ab9..3a6ff7e 100644 --- a/examples.md +++ b/examples.md @@ -53,12 +53,45 @@ This file contains examples of how to use the VFB3-MCP server tools. "arguments": { "query": "medulla", "filter_types": ["neuron", "adult"], - "exclude_types": ["deprecated"] + "boost_types": ["has_image"] } } } ``` +Deprecated terms are already excluded server-side, so there is no need to list them in +`exclude_types`. Get valid type names from `list_search_facets` rather than guessing. + +### 4b. List Valid Facet Type Names +```json +{ + "method": "tools/call", + "params": { + "name": "list_search_facets", + "arguments": { + "contains": "neuron" + } + } +} +``` + +### 4c. One Row Per Matching Synonym +```json +{ + "method": "tools/call", + "params": { + "name": "search_terms", + "arguments": { + "query": "Kenyon cell", + "unique": false + } + } +} +``` + +Useful when you need to know *which* name matched. The same `short_form` will repeat +across rows; the default `unique: true` gives one row per term instead. + ### 5. Search Terms with Minimization ```json { @@ -160,16 +193,48 @@ This file contains examples of how to use the VFB3-MCP server tools. "params": { "name": "query_connectivity", "arguments": { - "upstream_type": "FBbt_00000001", - "downstream_type": "FBbt_00000002", + "upstream_type": "FBbt_00003789", + "downstream_type": "FBbt_00003730", "weight": 5, "group_by_class": true, - "exclude_dbs": ["hemibrain"] + "exclude_dbs": ["mc"] } } } ``` +Tm1 → T3 across every connectome dataset except male-CNS: 11,916 connections +unfiltered, 7,309 with `mc` excluded. At least one of `upstream_type` / +`downstream_type` is required. + +`exclude_dbs` takes dataset **symbols**, and an unrecognised symbol is silently +ignored rather than reported — `exclude_dbs: ["male-cns"]` or `["hemibrain"]` excludes +nothing and says nothing. Call `list_connectome_datasets` and use the `symbol` field: +`BANC`, `fw`, `ol`, `mv`, `hb`, `mc`, `fafb`, `l1em`. + +Results come back as a strongest-first page plus a `summary` computed over every +connection found, so `count` is the true total and `returned` is only what you were +given. Broad queries find tens of thousands of connections. + +### 12b. Paging Through Connectivity Results +```json +{ + "method": "tools/call", + "params": { + "name": "query_connectivity", + "arguments": { + "upstream_type": "FBbt_00003789", + "weight": 50, + "limit": 25, + "offset": 25 + } + } +} +``` + +Rows 25–50 of the strongest-first ranking. The `summary` is identical on every page +because it always covers the full result set — answer from it rather than from the rows. + ### 13. Get Ontology Hierarchy Traverse `part_of` (region structure) or `subclass_of` (cell-type taxonomy) for a VFB term. `relationship` is required; `direction` defaults to `both` and `max_depth` to `1`. diff --git a/src/index.ts b/src/index.ts index 53d08dd..f388771 100644 --- a/src/index.ts +++ b/src/index.ts @@ -183,7 +183,7 @@ function setupToolHandlers(server: Server, sessionIdHolder?: RequestContext) { }, { name: 'search_terms', - description: 'Search VFB terms (Solr). USE filter_types BY DEFAULT — unfiltered searches return deprecated terms, scRNAseq artifacts, and developmental stages mixed in with the entity the user wants.\n\nCommon filter_types recipes:\n- Neuron classes: ["neuron", "class"]\n- Individual neurons with images: ["neuron", "has_image"]\n- Neurons with connectome data: ["neuron", "has_neuron_connectivity"]\n- Brain regions / neuropils: ["anatomy"]\n- Genes: ["gene"]\n- Driver lines / expression patterns: ["expression_pattern"]\n- Datasets: ["dataset"]\nAdd exclude_types: ["deprecated"] to almost any search to remove obsolete entities.\n\nStage filtering: VFB covers adult, larval, and embryonic data, and many anatomical FBbt classes are stage-agnostic. Do NOT add "adult" or "larva" to filter_types by default — only add them when the user is explicit about a stage (e.g. "adult Kenyon cells", "larval mushroom body"). Default searches should leave stage out so stage-agnostic classes and all life stages are visible.\n\nUseful flags:\n- minimize_results=true → top 10 + truncation metadata, for exploratory searches.\n- auto_fetch_term_info=true → if an exact label match is found, returns get_term_info in the same response.\n- boost_types=["has_image", "has_neuron_connectivity"] → soft-rank data-rich entities first without excluding others.\n\nIf the search returns no good matches, do NOT fall back to training-data answers — try alternative spellings, synonyms, broader terms, or different filter_types.\n\nMultiple filter_types are ANDed (results must match ALL). Multiple exclude_types are ORed (any match excludes). boost_types soft-rank without excluding.\n\nAvailable filter types: entity, anatomy, nervous_system, individual, has_image, adult, cell, neuron, vfb, has_neuron_connectivity, nblast, visual_system, cholinergic, class, secondary_neuron, expression_pattern, gabaergic, expression_pattern_fragment, glutamatergic, feature, sensory_neuron, neuronbridge, deprecated, larva, has_region_connectivity, nblastexp, gene, primary_neuron, flycircuit, mechanosensory_system, histaminergic, lineage_mbp, peptidergic, hasscrnaseq, chemosensory_system, split, has_subclass, olfactory_system, dopaminergic, fafb, l1em, pub, enzyme, motor_neuron, cluster, lineage_6, lineage_3, serotonergic, lineage_19, lineage_cm3, lineage_dm6, proprioceptive_system, gustatory_system, sense_organ, lineage_mbp4, lineage_mbp1, lineage_1, lineage_mbp2, lineage_all1, lineage_balc, lineage_cm4, lineage_dm4, muscle, lineage_13, lineage_8, lineage_mbp3, lineage_12, lineage_dm1, lineage_dpmm1, lineage_9, lineage_cp2, lineage_dl1, fanc, lineage_7, lineage_vpnd2, lineage_dm3, lineage_dpmpm2, lineage_14, lineage_4, lineage_blp1, lineage_dalv2, lineage_eba1, lineage_dm2, lineage_dpmpm1, auditory_system, lineage_16, lineage_blvp1, lineage_blav2, lineage_vlpl2, lineage_alad1, lineage_bamv3, lineage_bld6, lineage_vpnd1, synaptic_neuropil, lineage_23, lineage_17, lineage_10, lineage_dplpv, lineage_21, lineage_alv1\n\nMultiple filter_types are ANDed (results must match ALL). Multiple exclude_types are ORed (any match excludes). boost_types soft-rank matching results higher without excluding others.', + description: 'Search VFB terms. This is the search virtualflybrain.org itself runs — the same Solr query, the same ranking — so what comes back first here is what a user would see first on the site.\n\nUSE filter_types BY DEFAULT. Unfiltered searches mix scRNAseq artifacts and developmental stages in with the entity the user wants.\n\nCommon filter_types recipes:\n- Neuron classes: ["neuron", "class"]\n- Individual neurons with images: ["neuron", "has_image"]\n- Neurons with connectome data: ["neuron", "has_neuron_connectivity"]\n- Brain regions / neuropils: ["anatomy"]\n- Genes: ["gene"]\n- Driver lines / expression patterns: ["expression_pattern"]\n- Datasets: ["dataset"]\n\nThere are over 200 type names and they change as data is added, so do NOT guess them: call list_search_facets to see the current vocabulary (optionally filtered, e.g. contains="lineage"). Names are matched case- and separator-insensitively, and a name that does not exist is an error with suggestions rather than a silently empty result.\n\nDeprecated terms are excluded by the search itself — you do not need exclude_types: ["deprecated"], and adding it is harmless but pointless.\n\nStage filtering: VFB covers adult, larval, and embryonic data, and many anatomical FBbt classes are stage-agnostic. Do NOT add "adult" or "larva" to filter_types by default — only add them when the user is explicit about a stage (e.g. "adult Kenyon cells", "larval mushroom body"). Default searches should leave stage out so stage-agnostic classes and all life stages are visible.\n\nUseful flags:\n- unique=true (the default) → one row per term. Turn it OFF only when you need to see WHICH synonym matched; with unique=false a term appears once per matching synonym, so "Kenyon cell" can return the same ID several times.\n- minimize_results=true → top 10, essential fields only, for exploratory searches.\n- auto_fetch_term_info=true → if an exact label match is found, returns get_term_info in the same response.\n- boost_types=["has_image", "has_neuron_connectivity"] → float data-rich entities to the top of the list without excluding anything else.\n- demote_types=["expression_pattern_fragment"] → sink noisy types to the bottom of the list instead of removing them.\n\nIf the search returns no good matches, do NOT fall back to training-data answers — try alternative spellings, synonyms, broader terms, or different filter_types.\n\nMultiple filter_types are ANDed (results must match ALL). Multiple exclude_types are ORed (any match excludes). boost_types and demote_types re-order without excluding; boost wins if a term matches both.', inputSchema: { type: 'object', properties: { @@ -194,17 +194,27 @@ function setupToolHandlers(server: Server, sessionIdHolder?: RequestContext) { filter_types: { type: 'array', items: { type: 'string' }, - description: 'Filter results to only include items matching ALL of these facets_annotation types (AND logic)', + description: 'Filter results to only include items matching ALL of these facets_annotation types (AND logic). Use list_search_facets for valid names.', }, exclude_types: { type: 'array', items: { type: 'string' }, - description: 'Exclude results matching ANY of these facets_annotation types (OR logic)', + description: 'Exclude results matching ANY of these facets_annotation types (OR logic). Deprecated terms are already excluded.', }, boost_types: { type: 'array', items: { type: 'string' }, - description: 'Boost ranking of results matching these facets_annotation types without excluding others', + description: 'Float results matching these facets_annotation types to the top of the ranked list without excluding others', + }, + demote_types: { + type: 'array', + items: { type: 'string' }, + description: 'Sink results matching these facets_annotation types to the bottom of the ranked list without excluding them. Ignored for a type that also appears in boost_types.', + }, + unique: { + type: 'boolean', + description: 'One row per term (default true). Set false to get a row per matching synonym, which shows WHICH name matched at the cost of repeating IDs.', + default: true, }, start: { type: 'number', @@ -219,7 +229,7 @@ function setupToolHandlers(server: Server, sessionIdHolder?: RequestContext) { }, minimize_results: { type: 'boolean', - description: 'When true, limit results to top 10 for initial searches and add truncation metadata. For exact matches, return only the matching result.', + description: 'When true, return at most 10 results with only the essential fields. For exact matches, return only the matching result.', default: false, }, auto_fetch_term_info: { @@ -231,6 +241,19 @@ function setupToolHandlers(server: Server, sessionIdHolder?: RequestContext) { required: ['query'], }, }, + { + name: 'list_search_facets', + description: 'List the type names search_terms can filter, exclude, boost or demote by, with the number of terms carrying each one. Call this instead of guessing: there are over 200 names, they are the index\'s own annotations rather than a curated list, and they change as data is added. Use contains to narrow (e.g. contains="lineage" for the ~120 lineage clones, contains="connectivity" to find the connectome facets). The counts tell you whether a name is broad or niche — "entity" covers everything, a single lineage covers a handful.', + inputSchema: { + type: 'object', + properties: { + contains: { + type: 'string', + description: 'Only return type names containing this text. Matched case- and separator-insensitively, so "nervous system" finds "Nervous_system".', + }, + }, + }, + }, { name: 'resolve_entity', description: 'Resolve an unresolved FlyBase-related query string into VFB/FlyBase IDs and metadata. Pass the raw text exactly as the user wrote it (for example "P{VT054895-GAL4.DBD}", "Hb9-GAL4", "SS04495", "MB002B", "PAM cluster", or "dpp"). Do NOT pass resolved IDs such as FBgn/FBal/FBti/FBco/FBst or VFB IDs; if you already have an ID, use the downstream tool directly. Uses tiered resolution: exact name → synonym → broad pattern match. Returns match_type (EXACT/SYNONYM/BROAD), feature ID, name, type, and synonyms. IMPORTANT: When match_type is SYNONYM or BROAD, always confirm the resolved entity with the user before proceeding to further queries. If multiple matches are returned, show a disambiguation list and ask the user to choose. This tool queries FlyBase Chado — for VFB ontology lookups (anatomical terms, neuron class IDs) use search_terms instead.', @@ -269,7 +292,7 @@ function setupToolHandlers(server: Server, sessionIdHolder?: RequestContext) { }, { name: 'query_connectivity', - description: 'Query synaptic connectivity between Drosophila neuron classes across ALL connectome datasets simultaneously for comparative connectomics. This is NOT pre-cached — it runs live queries, so expect slow responses (up to several minutes). Set both upstream_type AND downstream_type to filter connections between two specific neuron classes (e.g., "What Tm1→T3 connections exist across all datasets?"). At least one of upstream_type or downstream_type is required. CONSTRAINTS: Only accepts neuron class terms (OWL IDs like FBbt_00003789 or labels like "transmedullary neuron Tm1") — anatomical regions or neuropils (e.g., "lobula", "medulla") are NOT accepted. NOT suitable for individual neuron-to-neuron connections — for pre-computed connections of a single individual neuron, use run_query with NeuronNeuronConnectivityQuery instead. NOT for muscle/sense organ connections. RECOMMENDED DEFAULTS: weight=5, exclude_dbs=["hb","fafb"] unless user specifies otherwise. For both-ends queries, start with weight≥50 to avoid timeouts. WORKFLOW: Confirm parameters with user before querying. Use search_terms with filter_types ["neuron","class"] to validate/canonicalize neuron type labels. If zero results, try relaxation: lower weight to 1, then remove exclude_dbs filter, then try group_by_class=true — report what worked and let user decide. Present large results (>50 rows) as top 20 by weight with summary stats.', + description: 'Query synaptic connectivity between Drosophila neuron classes across ALL connectome datasets simultaneously for comparative connectomics. This is NOT pre-cached — it runs live queries, so expect slow responses (up to several minutes). Set both upstream_type AND downstream_type to filter connections between two specific neuron classes (e.g., "What Tm1→T3 connections exist across all datasets?"). At least one of upstream_type or downstream_type is required. CONSTRAINTS: Only accepts neuron class terms (OWL IDs like FBbt_00003789 or labels like "transmedullary neuron Tm1") — anatomical regions or neuropils (e.g., "lobula", "medulla") are NOT accepted. NOT suitable for individual neuron-to-neuron connections — for pre-computed connections of a single individual neuron, use run_query with NeuronNeuronConnectivityQuery instead. NOT for muscle/sense organ connections. RECOMMENDED DEFAULTS: weight=5, exclude_dbs=["hb","fafb"] unless user specifies otherwise. For both-ends queries, start with weight≥50 to avoid timeouts. RESULT SIZE: a broad query is enormous (a single class at weight=5 can be over 50,000 connections), so results are ranked strongest-first and paged — you get limit rows (default 50) plus a summary computed over ALL of them: totals, per-dataset counts, distinct neuron counts, and the top class pairs. Answer from the summary and quote a handful of rows; only page with offset if the user asks for specific further rows. WORKFLOW: Confirm parameters with user before querying. Use search_terms with filter_types ["neuron","class"] to validate/canonicalize neuron type labels. If zero results, try relaxation: lower weight to 1, then remove exclude_dbs filter, then try group_by_class=true — report what worked and let user decide. group_by_class=true is usually the better first call on a broad query: it aggregates to class pairs instead of returning every neuron pair.', inputSchema: { type: 'object', properties: { @@ -292,7 +315,17 @@ function setupToolHandlers(server: Server, sessionIdHolder?: RequestContext) { exclude_dbs: { type: 'array', items: { type: 'string' }, - description: 'Dataset symbols to exclude (recommended default: ["hb", "fafb"] to focus on newer datasets). Pass empty array [] to include all datasets. Use list_connectome_datasets to see valid symbols.', + description: 'Dataset symbols to exclude (recommended default: ["hb", "fafb"] to focus on newer datasets). Pass empty array [] to include all datasets. Must be the exact `symbol` field from list_connectome_datasets — currently BANC, fw, ol, mv, hb, mc, fafb, l1em. An unrecognised symbol is silently ignored by the server rather than reported, so a dataset name ("hemibrain", "male-cns", "flywire") excludes nothing and gives no warning. Call list_connectome_datasets rather than guessing.', + }, + limit: { + type: 'number', + description: 'How many connection rows to return, strongest first (default 50). The summary always covers every connection found, not just the returned rows. Pass 0 for all rows — only do this on a query you already know is small, as broad queries return tens of thousands.', + default: 50, + }, + offset: { + type: 'number', + description: 'Row to start from within the strongest-first ranking (default 0). Re-running with the same limit and the next offset walks down the list.', + default: 0, }, }, }, @@ -350,7 +383,9 @@ function setupToolHandlers(server: Server, sessionIdHolder?: RequestContext) { case 'run_query': return await handleRunQuery(args as { id?: string | string[]; query_type?: string; queries?: Array<{ id: string; query_type: string }>; limit?: number; offset?: number; include_images?: boolean }); case 'search_terms': - return await handleSearchTerms(args as { query: string; filter_types?: string[]; exclude_types?: string[]; boost_types?: string[]; start?: number; rows?: number; minimize_results?: boolean; auto_fetch_term_info?: boolean }); + return await handleSearchTerms(args as { query: string; filter_types?: string[]; exclude_types?: string[]; boost_types?: string[]; demote_types?: string[]; unique?: boolean; start?: number; rows?: number; minimize_results?: boolean; auto_fetch_term_info?: boolean }); + case 'list_search_facets': + return await handleListSearchFacets(args as { contains?: string }); case 'resolve_entity': return await handleResolveEntity(args as { name: string }); case 'resolve_combination': @@ -358,7 +393,7 @@ function setupToolHandlers(server: Server, sessionIdHolder?: RequestContext) { case 'list_connectome_datasets': return await handleListConnectomeDatasets(); case 'query_connectivity': - return await handleQueryConnectivity(args as { upstream_type?: string; downstream_type?: string; weight?: number; group_by_class?: boolean; exclude_dbs?: string[] }); + return await handleQueryConnectivity(args as { upstream_type?: string; downstream_type?: string; weight?: number; group_by_class?: boolean; exclude_dbs?: string[]; limit?: number; offset?: number }); case 'get_hierarchy': return await handleGetHierarchy(args as { id: string; relationship: string; direction?: string; max_depth?: number }); default: @@ -602,141 +637,295 @@ async function handleRunQuery(args: { id?: string | string[]; query_type?: strin }; } -async function handleSearchTerms(args: { query: string; filter_types?: string[]; exclude_types?: string[]; boost_types?: string[]; start?: number; rows?: number; minimize_results?: boolean; auto_fetch_term_info?: boolean }) { - const { query, filter_types, exclude_types, boost_types, start = 0, rows = 150, minimize_results = false, auto_fetch_term_info = false } = args; - const baseUrl = 'https://solr.virtualflybrain.org/solr/ontology/select'; +// --------------------------------------------------------------------------- +// search_terms +// +// This used to build its own Solr query — its own fq, its own bq, its own qf — +// against solr.virtualflybrain.org. That construction was a copy of the +// website's, made once and then left to drift, and because it skipped the +// website's refine/sort pass its ordering was never actually the website's +// either. It now calls VFBquery's /search, which IS the website's search: same +// filters, same boosts, same comparator. One ranking, one place to fix it. +// --------------------------------------------------------------------------- - const fq: string[] = [ - '(short_form:VFB* OR short_form:FB* OR facets_annotation:DataSet OR facets_annotation:pub) AND NOT short_form:VFBc_*', - ]; +// How many candidates /search asks Solr for. 500 is the website's value, and it +// affects *ranking* rather than page size — the comparator can only promote what +// was retrieved — so it is not something to shrink for a small page. +const SEARCH_CANDIDATE_POOL = 500; + +// null until the first `unique` request answers it. The deployed service +// silently ignores query params it does not know, so the only way to tell +// whether it applied `unique` is whether it echoes the flag back. +// +// null is treated as "probably yes": every currently deployed VFBquery honours +// `unique`, so paying the un-truncated fetch on the first search of every +// process to re-prove that is a 600KB tax on the common case. A server that +// turns out not to echo the flag gets one refetch, then the answer is cached +// here. +// +// The answer can flip mid-process, which is why this is re-read rather than +// frozen: VFBQUERY_BASE sits behind an nginx cache, so a URL first requested +// before `unique` shipped keeps answering with the old body (no `unique`, no +// `distinct_terms`) while a novel URL answers with the new one. +let searchUniqueIsServerSide: boolean | null = null; + +function dedupeSearchRows(rows: any[]): any[] { + const seen = new Set(); + const out: any[] = []; + for (const row of rows) { + const sf = row?.short_form; + // A row with no short_form cannot be proven a duplicate, so it is kept. + if (typeof sf !== 'string' || sf === '') { out.push(row); continue; } + if (seen.has(sf)) { continue; } + seen.add(sf); + out.push(row); + } + return out; +} - if (filter_types && filter_types.length > 0) { - for (const ft of filter_types) { - fq.push(`facets_annotation:${ft}`); +// VFBquery's handlers explain their own rejections in the body — an unknown type +// name with suggestions, "At least one of upstream_type or downstream_type", a +// bad `relationship`. Some return JSON and some plain text, and passing the bare +// AxiosError up says only "status code 400", which tells the caller nothing about +// what to change. +function rejectionDetail(error: any): string | null { + const data = error?.response?.data; + if (data == null) return null; + if (typeof data === 'string') return data.trim() || null; + return data.error ?? data.detail ?? null; +} + +async function fetchSearch(params: URLSearchParams): Promise { + const url = `${VFBQUERY_BASE}/search?${params.toString()}`; + console.error(`MCP Debug: search ${params.toString()}`); + const response = await axios.get(url, { timeout: 120000 }); + return response.data; +} + +async function handleSearchTerms(args: { + query: string; + filter_types?: string[]; + exclude_types?: string[]; + boost_types?: string[]; + demote_types?: string[]; + unique?: boolean; + start?: number; + rows?: number; + minimize_results?: boolean; + auto_fetch_term_info?: boolean; +}) { + const { query, filter_types, exclude_types, boost_types, demote_types, + minimize_results = false, auto_fetch_term_info = false } = args; + + const start = Math.max(0, Math.trunc(Number(args.start) || 0)); + const requestedRows = Number.isFinite(Number(args.rows)) && Number(args.rows) > 0 + ? Math.min(1000, Math.trunc(Number(args.rows))) + : 150; + const rows = minimize_results ? Math.min(requestedRows, 10) : requestedRows; + const wantUnique = args.unique !== false; + + const params = new URLSearchParams({ query }); + // Grow the pool only when the caller pages past it, so an ordinary request + // gets the website's ranking rather than a differently-ranked wider net. + const pool = Math.min(1000, Math.max(SEARCH_CANDIDATE_POOL, start + rows)); + params.set('rows', String(pool)); + if (filter_types && filter_types.length) { params.set('filter_types', filter_types.join(',')); } + if (exclude_types && exclude_types.length) { params.set('exclude_types', exclude_types.join(',')); } + if (boost_types && boost_types.length) { params.set('boost_types', boost_types.join(',')); } + if (demote_types && demote_types.length) { params.set('demote_types', demote_types.join(',')); } + if (wantUnique) { params.set('unique', 'true'); } + + // Truncating server-side keeps the payload small, but only when the server + // collapses synonym rows itself; otherwise the rows past the limit are + // exactly the ones that would survive de-duplication here. + if (!wantUnique || searchUniqueIsServerSide !== false) { + params.set('limit', String(start + rows)); + } + + let data: any; + try { + data = await fetchSearch(params); + if (wantUnique) { + const echoed = typeof data?.unique === 'boolean'; + searchUniqueIsServerSide = echoed; + if (!echoed && params.has('limit')) { + // Deployed VFBquery predates `unique`. Page over the whole ranked list + // and collapse it here instead of guessing how far the duplicates run. + params.delete('limit'); + data = await fetchSearch(params); + } + } + } catch (error: any) { + // A 400 from /search carries the reason and, for a misspelled type name, + // the suggestions — far more useful to pass through than the status code. + const detail = error?.response?.data?.error; + if (detail) { + return { content: [{ type: 'text', text: `Search rejected: ${detail}` }] }; } + return { content: [{ type: 'text', text: `Error searching terms: ${error}` }] }; } - if (exclude_types && exclude_types.length > 0) { - const excludeClause = exclude_types.map(et => `facets_annotation:${et}`).join(' OR '); - fq.push(`NOT (${excludeClause})`); + const allRows: any[] = Array.isArray(data?.rows) ? data.rows : []; + const serverUnique = data?.unique === true; + const collapsed = wantUnique && !serverUnique ? dedupeSearchRows(allRows) : allRows; + const haveWholeList = !params.has('limit'); + + // `count` is the length of the list the server was paging through, so it + // already reflects `unique` when the server applied it. + let total: number; + if (wantUnique && !serverUnique) { + total = collapsed.length; + } else { + total = typeof data?.count === 'number' ? data.count : collapsed.length; + } + const distinctTerms = typeof data?.distinct_terms === 'number' + ? data.distinct_terms + : (haveWholeList ? dedupeSearchRows(allRows).length : undefined); + const solrMatches = typeof data?.solr_num_found === 'number' ? data.solr_num_found : undefined; + + // The exact-match shortcut only fires on the first page: pages 2+ are a + // caller walking the list, and collapsing that to one row loses their place. + const queryLower = String(query || '').trim().toLowerCase(); + let exactMatch: any = null; + if ((minimize_results || auto_fetch_term_info) && start === 0) { + exactMatch = collapsed.find((row: any) => + typeof row?.original_label === 'string' && row.original_label.toLowerCase() === queryLower + ) || null; } - let bq = 'short_form:VFBexp*^10.0 short_form:VFB*^100.0 short_form:FBbt*^100.0 short_form:FBbt_00003982^2 facets_annotation:Deprecated^0.001'; - if (boost_types && boost_types.length > 0) { - const boostClauses = boost_types.map(bt => `facets_annotation:${bt}^1000.0`).join(' '); - bq = `${bq} ${boostClauses}`; + let page: any[] = exactMatch ? [exactMatch] : collapsed.slice(start, start + rows); + if (minimize_results) { + page = page.map((row: any) => ({ + short_form: row?.short_form, + label: row?.label, + original_label: row?.original_label, + })); } - const params = { - q: `${query} OR ${query}* OR *${query}*`, - 'q.op': 'OR', - defType: 'edismax', - mm: '45%', - qf: 'label^110 synonym^100 label_autosuggest synonym_autosuggest shortform_autosuggest', - indent: 'true', - fl: 'short_form,label,synonym,id,facets_annotation,unique_facets', - start: start.toString(), - pf: 'true', - fq, - rows: Math.min(rows, 1000).toString(), // Cap at 1000 max - wt: 'json', - bq, + const notes: string[] = []; + if (exactMatch) { + notes.push(`"${query}" is an exact label match, so only that term is shown out of ${total}. Re-run with minimize_results=false and auto_fetch_term_info=false to see the rest.`); + } else if (total > start + page.length) { + notes.push(`Showing results ${start}-${start + page.length} of ${total}. To see more, re-run with start=${start + rows} (same rows).`); + } else if (start > 0) { + notes.push(`Showing results ${start}-${start + page.length} of ${total}.`); + } + if (minimize_results) { + notes.push('Only short_form, label and original_label are shown; re-run with minimize_results=false for facets and IDs.'); + } + if (wantUnique && !serverUnique) { + notes.push('The VFBquery response did not report server-side unique — either an older deployment, or a response cached from before it shipped — so duplicate synonym rows were collapsed by this MCP server instead. Ranking is unaffected; total counts terms.'); + } + if (solrMatches !== undefined && solrMatches > pool) { + notes.push(`Solr matched ${solrMatches} terms but only the top ${pool} were ranked, so this is not an exhaustive list. Narrow the query or add filter_types rather than paging past ${pool}.`); + } + // Under unique=false, total counts synonym rows while solr_matches counts terms, so + // total can exceed solr_matches. Read side by side that looks like a contradiction; + // it is just two different units, and saying so is cheaper than the reader guessing. + if (!wantUnique && solrMatches !== undefined && total > solrMatches) { + notes.push(`total (${total}) counts synonym rows because unique=false, while solr_matches (${solrMatches}) counts terms — ${solrMatches} terms matched under ${total} names between them. Re-run with unique=true (the default) for one row per term.`); + } + if (page.length === 0) { + notes.push('No matches. Try alternative spellings, a synonym, a broader term, or fewer filter_types — an empty result does not mean the entity does not exist in VFB.'); + } + + const result: Record = { + query, + unique: wantUnique, + start, + returned: page.length, + total, }; + if (distinctTerms !== undefined) { result.distinct_terms = distinctTerms; } + if (solrMatches !== undefined) { result.solr_matches = solrMatches; } + result.candidate_pool = pool; + if (notes.length) { result._note = notes.join(' '); } + result.results = page; - try { - const response = await axios.get(baseUrl, { params }); - let resultData = response.data; - - // Handle minimization and auto-fetch logic - if (minimize_results || auto_fetch_term_info) { - if (resultData?.response?.docs) { - const originalCount = resultData.response.numFound; - const queryLower = query.toLowerCase(); - const isPaginatedRequest = start > 0 || rows !== 150; - - // Check for exact label match first (only for non-paginated requests) - let exactMatch = null; - if (!isPaginatedRequest) { - exactMatch = resultData.response.docs.find((doc: any) => - doc.label?.toLowerCase() === queryLower - ); - } - - let minimizedDocs = resultData.response.docs; - let truncationInfo: any = {}; - - if (exactMatch) { - // If exact match found, return only that one - minimizedDocs = [exactMatch]; - truncationInfo = { exactMatch: true, totalAvailable: originalCount }; - } else if (minimize_results && !isPaginatedRequest) { - // For initial searches without pagination, limit to top 10 - minimizedDocs = resultData.response.docs.slice(0, 10); - truncationInfo = { - truncated: originalCount > 10, - shown: minimizedDocs.length, - totalAvailable: originalCount, - canRequestMore: originalCount > 10 - }; - } else if (isPaginatedRequest) { - // For paginated requests, return all requested results - truncationInfo = { - paginated: true, - requested: rows, - returned: minimizedDocs.length, - totalAvailable: originalCount - }; - } - - // Keep only essential fields if minimizing - if (minimize_results) { - minimizedDocs = minimizedDocs.map((doc: any) => ({ - short_form: doc.short_form, - label: doc.label, - synonym: Array.isArray(doc.synonym) ? doc.synonym.slice(0, 1) : doc.synonym // Keep only first synonym - })); - } - - resultData.response.docs = minimizedDocs; - resultData.response.numFound = minimizedDocs.length; // Update count - - // Add truncation metadata - if (Object.keys(truncationInfo).length > 0) { - resultData.response._truncation = truncationInfo; - } - - // Auto-fetch term info for exact match - if (auto_fetch_term_info && exactMatch) { - try { - const termInfoResult = await handleGetTermInfo({ id: exactMatch.short_form }); - if (termInfoResult.content && termInfoResult.content[0]?.text) { - resultData._term_info = JSON.parse(termInfoResult.content[0].text); - } - } catch (termInfoError) { - console.error('Error auto-fetching term info:', termInfoError); - // Don't fail the search if term info fetch fails - } - } + if (auto_fetch_term_info && exactMatch?.short_form) { + try { + const termInfoResult = await handleGetTermInfo({ id: exactMatch.short_form }); + if (termInfoResult.content && termInfoResult.content[0]?.text) { + result.term_info = JSON.parse(termInfoResult.content[0].text); } + } catch (termInfoError) { + // A failed term-info fetch must not fail the search it was bolted onto. + console.error('Error auto-fetching term info:', termInfoError); } + } - return { - content: [ - { - type: 'text', - text: JSON.stringify(resultData, null, 2), - }, - ], - }; - } catch (error) { - return { - content: [ - { + return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] }; +} + +// --------------------------------------------------------------------------- +// list_search_facets +// +// The type names search_terms filters by are the index's own annotations, so +// the only honest source for them is the index. This list is what the tool +// description used to hardcode: a snapshot, already short of the live +// vocabulary by roughly half, kept solely as a fallback for services that +// predate /facets. It is not maintained. +// --------------------------------------------------------------------------- + +const STATIC_FACET_SNAPSHOT: string[] = [ + 'entity', 'anatomy', 'nervous_system', 'individual', 'has_image', 'adult', 'cell', 'neuron', + 'vfb', 'has_neuron_connectivity', 'nblast', 'visual_system', 'cholinergic', 'class', + 'secondary_neuron', 'expression_pattern', 'gabaergic', 'expression_pattern_fragment', + 'glutamatergic', 'feature', 'sensory_neuron', 'neuronbridge', 'deprecated', 'larva', + 'has_region_connectivity', 'nblastexp', 'gene', 'primary_neuron', 'flycircuit', + 'mechanosensory_system', 'histaminergic', 'lineage_mbp', 'peptidergic', 'hasscrnaseq', + 'chemosensory_system', 'split', 'has_subclass', 'olfactory_system', 'dopaminergic', 'fafb', + 'l1em', 'pub', 'enzyme', 'motor_neuron', 'cluster', 'lineage_6', 'lineage_3', 'serotonergic', + 'lineage_19', 'lineage_cm3', 'lineage_dm6', 'proprioceptive_system', 'gustatory_system', + 'sense_organ', 'lineage_mbp4', 'lineage_mbp1', 'lineage_1', 'lineage_mbp2', 'lineage_all1', + 'lineage_balc', 'lineage_cm4', 'lineage_dm4', 'muscle', 'lineage_13', 'lineage_8', + 'lineage_mbp3', 'lineage_12', 'lineage_dm1', 'lineage_dpmm1', 'lineage_9', 'lineage_cp2', + 'lineage_dl1', 'fanc', 'lineage_7', 'lineage_vpnd2', 'lineage_dm3', 'lineage_dpmpm2', + 'lineage_14', 'lineage_4', 'lineage_blp1', 'lineage_dalv2', 'lineage_eba1', 'lineage_dm2', + 'lineage_dpmpm1', 'auditory_system', 'lineage_16', 'lineage_blvp1', 'lineage_blav2', + 'lineage_vlpl2', 'lineage_alad1', 'lineage_bamv3', 'lineage_bld6', 'lineage_vpnd1', + 'synaptic_neuropil', 'lineage_23', 'lineage_17', 'lineage_10', 'lineage_dplpv', 'lineage_21', + 'lineage_alv1', +]; + +function normaliseFacetName(value: string): string { + return String(value || '').trim().toLowerCase().replace(/[\s_-]+/g, ''); +} + +async function handleListSearchFacets(args: { contains?: string }): Promise<{ content: Array<{ type: string; text: string }> }> { + const contains = (args?.contains || '').trim(); + const params = new URLSearchParams(); + if (contains) { params.set('contains', contains); } + const qs = params.toString(); + const url = `${VFBQUERY_BASE}/facets${qs ? `?${qs}` : ''}`; + console.error(`MCP Debug: list_search_facets contains=${contains || '(none)'}`); + try { + const response = await axios.get(url, { timeout: 60000 }); + return { content: [{ type: 'text', text: JSON.stringify(response.data, null, 2) }] }; + } catch (error: any) { + const status = error?.response?.status; + if (status === 404 || status === 503) { + const needle = normaliseFacetName(contains); + const names = needle + ? STATIC_FACET_SNAPSHOT.filter((n) => normaliseFacetName(n).includes(needle)) + : STATIC_FACET_SNAPSHOT; + return { + content: [{ type: 'text', - text: `Error searching terms: ${error}`, - }, - ], - }; + text: JSON.stringify({ + source: 'static snapshot bundled with this MCP server', + _note: status === 404 + ? 'The deployed VFBquery does not serve /facets yet, so this is a stale snapshot of about 100 names rather than the live vocabulary of over 200, and it has no term counts. Names outside it may still be valid: search_terms will tell you if one is not.' + : 'The live type-name vocabulary is temporarily unavailable, so this is a stale bundled snapshot. Search itself still works.', + count: names.length, + total: STATIC_FACET_SNAPSHOT.length, + contains: contains || null, + facets: names.map((name) => ({ name })), + }, null, 2), + }], + }; + } + return { content: [{ type: 'text', text: `Error listing search facets: ${error}` }] }; } } @@ -831,12 +1020,155 @@ async function handleListConnectomeDatasets(): Promise<{ content: Array<{ type: } } +// /query_connectivity has no paging of its own and returns every row it finds: +// a single class at weight=5 is over 50,000 connections, which is not something +// to hand an LLM whole. These shape it into a strongest-first page plus a +// summary computed over the full set, so the numbers stay true even though the +// rows are truncated. + +const CONNECTIVITY_WEIGHT_KEYS = ['weight', 'total_weight', 'pairwise_connections']; +const CONNECTIVITY_DEFAULT_LIMIT = 50; +const CONNECTIVITY_TOP_PAIRS = 10; + +function connectivityWeightKey(rows: any[]): string | null { + for (const key of CONNECTIVITY_WEIGHT_KEYS) { + if (rows.some((row) => row && typeof row[key] === 'number')) { return key; } + } + return null; +} + +// Class labels come out of Neo4j as apoc.text.join(collect(distinct c.label),'|'), +// and the order inside that join is not deterministic — the same logical pair of +// classes comes back as "A|B" on one row and "B|A" on the next. Left alone that +// splits one class pair across several summary entries and inflates +// distinct_class_pairs. Sorting the parts gives one stable spelling per set. +function canonicaliseClassLabel(value: any): any { + if (typeof value !== 'string' || !value.includes('|')) { return value; } + return value + .split('|') + .map((part) => part.trim()) + .filter((part) => part.length > 0) + .sort() + .join('|'); +} + +function summariseConnectivity(rows: any[], weightKey: string | null): Record { + const summary: Record = { connections: rows.length }; + + if (weightKey) { + let total = 0; + let min = Infinity; + let max = -Infinity; + let counted = 0; + for (const row of rows) { + const w = row?.[weightKey]; + if (typeof w !== 'number') { continue; } + total += w; + if (w < min) { min = w; } + if (w > max) { max = w; } + counted++; + } + if (counted > 0) { + summary[weightKey] = { min, max, total, mean: Math.round((total / counted) * 10) / 10 }; + } + } + + const datasets: Record = {}; + for (const row of rows) { + const ds = row?.up_data_source || row?.down_data_source; + if (typeof ds === 'string' && ds) { datasets[ds] = (datasets[ds] || 0) + 1; } + } + if (Object.keys(datasets).length) { summary.by_dataset = datasets; } + + const pairs = new Map(); + for (const row of rows) { + if (!row || (row.upstream_class === undefined && row.downstream_class === undefined)) { continue; } + const up = canonicaliseClassLabel(row.upstream_class); + const down = canonicaliseClassLabel(row.downstream_class); + const key = JSON.stringify([up, down]); + let entry = pairs.get(key); + if (!entry) { + entry = { upstream_class: up, downstream_class: down, connections: 0, weight: 0 }; + pairs.set(key, entry); + } + entry.connections++; + const w = weightKey ? row[weightKey] : undefined; + if (typeof w === 'number') { entry.weight += w; } + } + if (pairs.size) { + summary.distinct_class_pairs = pairs.size; + summary.top_class_pairs = [...pairs.values()] + .sort((a, b) => (b.weight - a.weight) || (b.connections - a.connections)) + .slice(0, CONNECTIVITY_TOP_PAIRS); + } + + const upstream = new Set(); + const downstream = new Set(); + for (const row of rows) { + if (row?.upstream_neuron_id) { upstream.add(row.upstream_neuron_id); } + if (row?.downstream_neuron_id) { downstream.add(row.downstream_neuron_id); } + } + if (upstream.size) { summary.distinct_upstream_neurons = upstream.size; } + if (downstream.size) { summary.distinct_downstream_neurons = downstream.size; } + + return summary; +} + +function shapeConnectivityResult(data: any, ctx: { limit: number; offset: number }): any { + if (!data || !Array.isArray(data.connections)) { return data; } + const all: any[] = data.connections; + const total = typeof data.count === 'number' ? data.count : all.length; + const weightKey = connectivityWeightKey(all); + + // Per-neuron rows arrive in Cypher's order, which is not ranked, so a + // truncated page of them would be arbitrary. Sorting by weight makes the + // first page the interesting end; class-aggregated rows are already ordered + // this way, so for those it changes nothing. Index carried as a tiebreak to + // keep the sort stable. + const ranked = weightKey + ? all + .map((row, i): [any, number] => [row, i]) + .sort((a, b) => { + const wa = typeof a[0]?.[weightKey] === 'number' ? a[0][weightKey] : -Infinity; + const wb = typeof b[0]?.[weightKey] === 'number' ? b[0][weightKey] : -Infinity; + return (wb - wa) || (a[1] - b[1]); + }) + .map(([row]) => row) + : all; + + const page = ctx.limit > 0 ? ranked.slice(ctx.offset, ctx.offset + ctx.limit) : ranked.slice(ctx.offset); + + const notes: string[] = []; + if (total > ctx.offset + page.length) { + const nextOffset = ctx.offset + (ctx.limit > 0 ? ctx.limit : page.length); + notes.push(`Showing connections ${ctx.offset}-${ctx.offset + page.length} of ${total}, ranked by ${weightKey || "the endpoint's own order"} (strongest first). The summary covers ALL ${total} connections, not just this page, so answer from it rather than from the rows shown. To see further rows, re-run with offset=${nextOffset} (same limit).`); + } else if (ctx.offset > 0) { + notes.push(`Showing connections ${ctx.offset}-${ctx.offset + page.length} of ${total}.`); + } + + const shaped: Record = { + count: total, + offset: ctx.offset, + limit: ctx.limit, + returned: page.length, + ranked_by: weightKey, + summary: summariseConnectivity(all, weightKey), + }; + if (notes.length) { shaped._note = notes.join(' '); } + if (Array.isArray(data.warnings) && data.warnings.length) { shaped.warnings = data.warnings; } + if (data.resolved !== undefined) { shaped.resolved = data.resolved; } + shaped.connections = page; + return shaped; +} + async function handleQueryConnectivity(args: { upstream_type?: string; downstream_type?: string; weight?: number; group_by_class?: boolean; exclude_dbs?: string[]; + limit?: number; + offset?: number; }): Promise<{ content: Array<{ type: string; text: string }> }> { const params = new URLSearchParams(); if (args.upstream_type) params.set('upstream_type', args.upstream_type); @@ -844,12 +1176,23 @@ async function handleQueryConnectivity(args: { if (args.weight !== undefined) params.set('weight', String(args.weight)); if (args.group_by_class !== undefined) params.set('group_by_class', String(args.group_by_class)); if (args.exclude_dbs) params.set('exclude_dbs', args.exclude_dbs.join(',')); + // limit/offset are applied here, not sent: the endpoint has no paging. + const limit = Number.isFinite(Number(args.limit)) && Number(args.limit) >= 0 + ? Math.trunc(Number(args.limit)) + : CONNECTIVITY_DEFAULT_LIMIT; + const offset = Number.isFinite(Number(args.offset)) && Number(args.offset) > 0 + ? Math.trunc(Number(args.offset)) + : 0; const url = `${VFBQUERY_BASE}/query_connectivity?${params.toString()}`; - console.error(`MCP Debug: query_connectivity params=${params.toString()}`); + console.error(`MCP Debug: query_connectivity params=${params.toString()} limit=${limit} offset=${offset}`); try { const response = await axios.get(url, { timeout: 300000 }); // 5 min — live cross-dataset query - return { content: [{ type: 'text', text: JSON.stringify(response.data, null, 2) }] }; + return { content: [{ type: 'text', text: JSON.stringify(shapeConnectivityResult(response.data, { limit, offset }), null, 2) }] }; } catch (error) { + const detail = rejectionDetail(error); + if (detail) { + return { content: [{ type: 'text', text: `Connectivity query rejected: ${detail}` }] }; + } return { content: [{ type: 'text', text: `Error querying connectivity: ${error}` }] }; } } @@ -862,7 +1205,10 @@ async function handleGetHierarchy(args: { }): Promise<{ content: Array<{ type: string; text: string }> }> { const params = new URLSearchParams(); params.set('id', args.id); - params.set('relationship', args.relationship); + // Omit rather than send "undefined": the schema requires `relationship`, but a + // caller that drops it should get the endpoint's own default (part_of) instead + // of a 400 on a literal string. + if (args.relationship) params.set('relationship', args.relationship); if (args.direction) params.set('direction', args.direction); if (args.max_depth !== undefined) params.set('max_depth', String(args.max_depth)); const url = `${VFBQUERY_BASE}/get_hierarchy?${params.toString()}`; @@ -871,6 +1217,10 @@ async function handleGetHierarchy(args: { const response = await axios.get(url, { timeout: 120000 }); // 2 min return { content: [{ type: 'text', text: JSON.stringify(response.data, null, 2) }] }; } catch (error) { + const detail = rejectionDetail(error); + if (detail) { + return { content: [{ type: 'text', text: `Hierarchy request rejected: ${detail}` }] }; + } return { content: [{ type: 'text', text: `Error getting hierarchy: ${error}` }] }; } } @@ -1062,11 +1412,13 @@ function getHtmlPage(): string {
  • get_term_info - Get term information from VirtualFlyBrain using a VFB ID
  • run_query - Run a query on VirtualFlyBrain using a VFB ID and query type
    • -
  • search_terms - Search for VFB terms using the Solr search server with filtering options
    • +
  • search_terms - Search VFB terms with the website's own ranking, plus type filters, boosts and demotions
    • +
  • list_search_facets - List the type names search_terms can filter, exclude, boost or demote by, with term counts
  • resolve_entity - Resolve an unresolved query string (e.g., P{VT054895-GAL4.DBD} or a driver line / cell type label) to VFB/FlyBase IDs and metadata
  • resolve_combination - Resolve an unresolved split-GAL4 combination name or synonym to its underlying IDs
  • list_connectome_datasets - List available connectome datasets (e.g., Hemibrain, FAFB)
    • -
  • query_connectivity - Query connectivity across connectome datasets using upstream/downstream filters
    • +
  • query_connectivity - Query connectivity across connectome datasets using upstream/downstream filters, paged with summary statistics
    • +
  • get_hierarchy - Build a part_of or subclass_of hierarchy tree for a VFB term

🧠 About VirtualFlyBrain