diff --git a/config.toml b/config.toml index e79ca49..e768e80 100644 --- a/config.toml +++ b/config.toml @@ -109,7 +109,10 @@ section = ["HTML", "print", "RSS"] [privacy.disqus] disable = false [privacy.googleAnalytics] - anonymizeIP = false + # anonymizeIP is a no-op for GA4 (Universal Analytics-era flag) but is set + # to true here to signal intent — GA4 itself transiently uses the IP for + # geolocation only and does not store it. See content/en/about/cookies.md. + anonymizeIP = true disable = false respectDoNotTrack = true useSessionStorage = true diff --git a/content/en/_index.html b/content/en/_index.html index a4b8987..1a9c07f 100644 --- a/content/en/_index.html +++ b/content/en/_index.html @@ -22,15 +22,15 @@

- Our colleagues at FlyBase needs your help! + Our colleagues at FlyBase need your help!

- Because of recent changes to government funding, the NIH grant that supported FlyBase has been terminated. - They are now reaching out to the community for emergency funding. Please click here for more information and to contribute: + Because of changes to US government funding, the foreign subaward of the NHGRI grant that supported the FlyBase curation team at the University of Cambridge has been terminated. + They are now reaching out to the community to assist with funding. Please click here for more information and to contribute: Contribute to FlyBase wiki page - Thank you in advance for your support to keep FlyBase on air and continuing to serve the Drosophila research community. + Thank you in advance for your support in allowing FlyBase to continue serving the Drosophila research community. We remain grateful to Wellcome for their continued support of Virtual Fly Brain.

diff --git a/content/en/about/accessibility.md b/content/en/about/accessibility.md new file mode 100644 index 0000000..08e2708 --- /dev/null +++ b/content/en/about/accessibility.md @@ -0,0 +1,74 @@ +--- +title: "Accessibility Statement" +linkTitle: "Accessibility" +weight: 22 +--- + +## Accessibility statement for Virtual Fly Brain + +Website accessibility statement in line with the Public Sector Bodies (Websites and Mobile Applications) (No. 2) Accessibility Regulations 2018. + +This statement applies to the public Virtual Fly Brain (VFB) services hosted at [www.virtualflybrain.org](https://www.virtualflybrain.org) and its sub-domains. VFB is run by the [University of Edinburgh's School of Informatics](https://www.inf.ed.ac.uk/) as a research data integration platform for *Drosophila melanogaster* neurobiology. + +The University of Edinburgh is committed to making its websites and applications accessible in line with the regulations. This website is partially compliant with the [Web Content Accessibility Guidelines (WCAG) 2.2 AA standard](https://www.w3.org/TR/WCAG22/), with the non-compliances listed below. + +We want as many people as possible to be able to use this website. Where the underlying content is inherently visual (3D image stacks, neuron skeletons and connectivity diagrams) we recognise that some features cannot be made fully accessible without losing scientific meaning, and we have documented those limitations below. + +### Using this website + +Across the site you should be able to: + +- Change colours, contrast levels and font size using your browser settings on the documentation pages +- Use the documentation pages without encountering flashing, scrolling or moving content +- Listen to most of the documentation pages with a screen reader (JAWS, NVDA, VoiceOver) +- Navigate the documentation pages by keyboard alone +- Resize text up to 200% on the documentation pages without loss of content + +The 3D web client (Geppetto-based) and the embedded CATMAID connectome viewers have known accessibility limitations — see below. + +### Customising the website + +[AbilityNet — My Computer My Way](https://mcmw.abilitynet.org.uk/) has advice on making your device easier to use if you have a disability. + +### Known accessibility issues + +#### Non-compliance with WCAG 2.2 AA criteria + +- **1.1.1 Non-text content.** Not all neuron images and anatomical diagrams have meaningful alternative text. +- **1.3.1 Info and relationships, 1.3.2 Meaningful sequence.** Some legacy figure pages do not present content in a programmatically determined reading order. +- **1.4.1 Use of colour.** Some neuroanatomical schematics use colour as the sole means of distinguishing regions or cell types. +- **1.4.3 Contrast (minimum).** Some embedded figures from external publications use low-contrast labels we cannot modify. +- **2.1.1 Keyboard.** The Geppetto 3D viewer cannot be navigated by keyboard alone for full feature parity with mouse interaction. +- **4.1.2 Name, role, value.** The 3D viewer's scene-graph elements (neurons, brain regions) are not exposed to assistive technology as named, role-tagged elements. + +#### Content that is not within the scope of the accessibility regulations + +- **Third-party scientific content.** PDFs and figures imported from peer-reviewed publications retain their original formatting, including small text size and low colour contrast, and are republished under their original licences. We do not modify them. +- **The CATMAID connectome viewers (`*.catmaid.virtualflybrain.org`)** are embedded third-party software whose interface is optimised for mouse and keyboard interaction at high zoom levels. We do not control its accessibility behaviour. +- **Older content (pre-September 2018).** Some legacy pages and PDFs pre-date the regulations and are kept for reference; we will replace them with accessible HTML where reasonable. + +#### Ongoing improvements + +- Adding `aria-label` and image-map fallbacks to anatomical schematics. +- Documenting alternative routes to the same information (text search, the [VFB_connect Python API](https://github.com/VirtualFlyBrain/VFB_connect), and the [VFB MCP server](https://virtualflybrain.org/blog/2026/02/07/introducing-the-vfb-model-context-protocol-mcp-tool/)) so users who cannot use the 3D viewer can still access the underlying data programmatically. +- Reviewing and improving alternative text for image content. +- Following the [University of Edinburgh accessibility guidance](https://www.ed.ac.uk/about/website/accessibility) for the documentation site framework. + +### Feedback and contact information + +If you need information from this site in a different format, or you encounter an accessibility problem we have not listed: + +- Email: [support@virtualflybrain.org](mailto:support@virtualflybrain.org) +- GitHub: file an issue at [github.com/VirtualFlyBrain/VFB2/issues](https://github.com/VirtualFlyBrain/VFB2/issues) + +We will respond within five working days. + +### Enforcement procedure + +The Equality and Human Rights Commission (EHRC) is responsible for enforcing the Public Sector Bodies (Websites and Mobile Applications) (No. 2) Accessibility Regulations 2018. If you are not satisfied with how we respond to your complaint, you can contact the [Equality Advisory and Support Service (EASS)](https://www.equalityadvisoryservice.com/). + +### Preparation of this statement + +This statement was prepared in May 2026, it has not been independently audited. + +This article was published on 2026-05-19. diff --git a/content/en/about/contributors.md b/content/en/about/contributors.md index 1e3601d..d6da132 100644 --- a/content/en/about/contributors.md +++ b/content/en/about/contributors.md @@ -71,6 +71,7 @@ img { [![actions-user](https://avatars.githubusercontent.com/u/65916846?v=4&s=100)](https://github.com/actions-user) [![NeZanyat](https://avatars.githubusercontent.com/u/1211953?v=4&s=100)](https://github.com/NeZanyat) [![afonsobspinto](https://avatars.githubusercontent.com/u/19196034?v=4&s=100)](https://github.com/afonsobspinto) +[![Copilot](https://avatars.githubusercontent.com/in/1143301?v=4&s=100)](https://github.com/apps/copilot-swe-agent) [![snyk-bot](https://avatars.githubusercontent.com/u/19733683?v=4&s=100)](https://github.com/snyk-bot) [![leechuck](https://avatars.githubusercontent.com/u/362161?v=4&s=100)](https://github.com/leechuck) [![MiguelAngelRG](https://avatars.githubusercontent.com/u/12188797?v=4&s=100)](https://github.com/MiguelAngelRG) diff --git a/content/en/about/cookies.md b/content/en/about/cookies.md new file mode 100644 index 0000000..357181f --- /dev/null +++ b/content/en/about/cookies.md @@ -0,0 +1,54 @@ +--- +title: "Cookies" +linkTitle: "Cookies" +weight: 21 +--- + +## Cookies used by Virtual Fly Brain + +**Last updated: May 2026** + +This page lists the cookies set by [www.virtualflybrain.org](https://www.virtualflybrain.org) and the related Virtual Fly Brain (VFB) services. It complements the [Privacy Notice](/about/privacy/). + +### What is a cookie? + +A cookie is a small text file that a website saves on your device when you visit. Cookies let a site recognise your browser between requests and on return visits. + +### Strictly necessary cookies + +These cookies are required for the website and applications to work and cannot be turned off through any opt-out mechanism without breaking core features. + +| Name | Set by | Purpose | Expires | +|------|--------|---------|---------| +| `JSESSIONID` | Geppetto (`v2.virtualflybrain.org` and variants) | Maintains your application session in the 3D web client — currently selected neurons, layer visibility and camera state. | End of browser session | + +### Performance cookies + +These cookies help us understand how visitors use Virtual Fly Brain so we can improve content, navigation and the underlying data integrations. The data is aggregated and is not used to identify individual users. + +#### Google Analytics 4 + +We use Google Analytics 4 (GA4), a web analytics service provided by Google LLC. GA4 receives your IP address transiently for geolocation purposes, derives an approximate location (city/country level), and discards the raw IP — it is not stored against your analytics record. The data we see is aggregated counts, paths through the site, and broad geographic distribution. + +| Name | Purpose | Expires | +|------|---------|---------| +| `_ga` | Distinguishes browsers across visits. | 2 years from your most recent visit | +| `_ga_K7DDZVVXM7` | Maintains GA4 session state for the VFB property. | 2 years from your most recent visit | + +For details of Google's processing, see [Google's Privacy & Terms](https://policies.google.com/privacy). + +International transfer note: Google may process GA4 data on servers in the United States. The transfer is covered by the EU-U.S. Data Privacy Framework (Commission Implementing Decision (EU) 2023/1795); Google is certified under the framework. The UK extension to the framework applies for UK-origin data. + +### How to opt out + +You can opt out of Google Analytics on every site you visit by installing Google's [Analytics Opt-out browser add-on](https://tools.google.com/dlpage/gaoptout). You can also clear or block individual cookies via your browser's settings; doing so will not prevent you from using the public VFB services, but the Geppetto session cookie above is needed for the 3D viewer. + +### Other cookies set by sites we link to + +Pages elsewhere on virtualflybrain.org may embed content from third-party services (for example, YouTube or X/Twitter). Those embeds may set their own cookies; please refer to the third party's own cookie or privacy policy for details. We do not set non-essential third-party cookies of our own beyond Google Analytics. + +### Contact + +Questions about cookies on this site: . + +Wider University data-protection enquiries: [University of Edinburgh Data Protection Officer](mailto:dpo@ed.ac.uk). diff --git a/content/en/about/privacy.md b/content/en/about/privacy.md index 21bcdb1..de4b3cb 100644 --- a/content/en/about/privacy.md +++ b/content/en/about/privacy.md @@ -6,22 +6,23 @@ weight: 20 ## Virtual Fly Brain Privacy Notice -**Last updated: February 2026** +**Last updated: May 2026** The Virtual Fly Brain (VFB) project is committed to protecting your privacy and keeping you informed about how your personal information is used. This privacy notice explains how we collect, use, and protect your personal data when you visit our website. ### What information do we collect? -We collect information about your visit to our website through Google Analytics, a web analytics service provided by Google, Inc. This includes: +**Website analytics (Google Analytics 4).** We use Google Analytics 4 to understand how the site is used. When you visit a page, your IP address is sent to Google for the purpose of deriving an approximate geographic location, and is then discarded; the raw IP is not stored against your analytics record. The data we receive is aggregated and includes: -- Your IP address (anonymized) +- Approximate geographic location (city/country level) - Browser type and version - Operating system - Referring website - Pages visited and time spent on each page -- Geographic location (country/city level only) -We do not collect any personally identifiable information such as names, email addresses, or contact details unless you voluntarily provide them (for example, through our contact forms or feedback mechanisms). +**Web-server logs.** Our front-end load balancers and application containers produce request logs that include your IP address. These logs are short-lived and operational only — they are not persisted beyond the lifetime of the running container and are lost on restart. + +We do not collect personally identifiable information such as names, email addresses or contact details unless you voluntarily provide them through our contact forms, feedback links or community channels. ### How do we use this information? @@ -31,13 +32,13 @@ The information collected helps us: - Improve the user experience and content - Identify technical issues - Analyze website traffic patterns -- Support our research by understanding user interests in neuroanatomical data +- Understand user interests in our neuroanatomical data -### Cookies and Website Privacy +### Cookies -We use Google Analytics cookies to collect this information. These are small text files stored on your device. You can control cookies through your browser settings, and you can opt out of Google Analytics tracking by visiting [Google Analytics Opt-out](https://tools.google.com/dlpage/gaoptout). +Google Analytics 4 sets cookies on your browser to distinguish your visits. The Geppetto 3D web client sets a session cookie to maintain application state during a visit. A full list of cookies set by Virtual Fly Brain, with their purposes and expiry, is on the [Cookies page](/about/cookies/), which also explains how to opt out of Google Analytics. -For more information about Google's privacy practices, please see [Google's Privacy Policy](https://policies.google.com/privacy). +For Google's processing details see [Google's Privacy & Terms](https://policies.google.com/privacy). ### Legal Basis for Processing @@ -48,7 +49,9 @@ Our processing of your personal data is based on: ### Data Sharing and Disclosure -We do not sell, rent, or trade your personal information. Information collected by Google Analytics may be processed by Google on servers located in the United States. Google is committed to complying with the EU-U.S. Privacy Shield framework. +We do not sell, rent or trade your personal information. + +Analytics data collected by Google Analytics may be processed by Google on servers located in the United States. International transfers are covered by the EU-U.S. Data Privacy Framework (Commission Implementing Decision (EU) 2023/1795); Google is [certified under the framework](https://www.dataprivacyframework.gov/list). The UK extension to the framework applies for UK-origin data. ### Data Retention diff --git a/content/en/blog/news/VFB_MCP_Tool.md b/content/en/blog/news/VFB_MCP_Tool.md index 592e2e8..1d0dc29 100644 --- a/content/en/blog/news/VFB_MCP_Tool.md +++ b/content/en/blog/news/VFB_MCP_Tool.md @@ -1,7 +1,7 @@ --- title: "Introducing the VFB Model Context Protocol (MCP) Tool" linkTitle: "VFB MCP Tool Launch" -date: 2026-02-07 +date: 2026-06-21 description: > Explore Virtual Fly Brain data directly through your favorite LLM with the new VFB Model Context Protocol (MCP) tool - now available at vfb3-mcp.virtualflybrain.org --- @@ -53,3 +53,9 @@ This tool democratizes access to VFB's complex neuroanatomical data, making it a Check out our new [**VFB MCP Tutorial**](/docs/tutorials/vfb-mcp-guide/) for detailed guidance on using the tool, example queries, and best practices for exploring VFB data through your favorite LLM. The VFB MCP tool represents an exciting new frontier in how researchers interact with neuroanatomical databases. We hope it accelerates your discoveries about *Drosophila* neurobiology! + +## Now described in a preprint + +The tool and its evaluation are now described in a preprint: McLachlan et al. (2026), *VFB-MCP: Natural-Language Access to Drosophila Neuroscience Grounded by an Expert-Curated Ontology-Led Knowledgebase*, bioRxiv. [doi:10.64898/2026.06.16.732577](https://doi.org/10.64898/2026.06.16.732577). + +Benchmarked on 30 neuroscience tasks, an LLM equipped with the VFB MCP produced precise, verifiable and appropriately quantified answers on 25/30 tasks, against 14/30 for a web-search-assisted LLM and 2/30 for a bare LLM (Wilcoxon *p* < 0.01, Holm-corrected across all pairwise comparisons). The advantage was largest where answers required data quantification — 89% of those tasks were answered correctly with the MCP, against 11% with web search — reflecting the precision that VFB's expert-curated, ontology-backed knowledge graph brings to the recently integrated connectomes. The MCP server code is available at [VirtualFlyBrain/VFB3-MCP](https://github.com/VirtualFlyBrain/VFB3-MCP). \ No newline at end of file diff --git a/content/en/docs/Anatomy Diagrams/Anatomy of the central complex/_index.md b/content/en/docs/Anatomy Diagrams/Anatomy of the central complex/_index.md index bbbeb58..2acd341 100644 --- a/content/en/docs/Anatomy Diagrams/Anatomy of the central complex/_index.md +++ b/content/en/docs/Anatomy Diagrams/Anatomy of the central complex/_index.md @@ -28,6 +28,8 @@ description: > + + diff --git a/content/en/docs/Anatomy Diagrams/Compartmentalization of mushroom body lobes/_index.md b/content/en/docs/Anatomy Diagrams/Compartmentalization of mushroom body lobes/_index.md index 55f0326..c420854 100644 --- a/content/en/docs/Anatomy Diagrams/Compartmentalization of mushroom body lobes/_index.md +++ b/content/en/docs/Anatomy Diagrams/Compartmentalization of mushroom body lobes/_index.md @@ -15,10 +15,10 @@ description: > - - - - + + + + diff --git a/content/en/docs/Anatomy Diagrams/Olfactory system neuroanatomy/_index.md b/content/en/docs/Anatomy Diagrams/Olfactory system neuroanatomy/_index.md index f8b254f..b2610fe 100644 --- a/content/en/docs/Anatomy Diagrams/Olfactory system neuroanatomy/_index.md +++ b/content/en/docs/Anatomy Diagrams/Olfactory system neuroanatomy/_index.md @@ -9,15 +9,15 @@ description: > - - + + + - diff --git a/vfbterms.py b/vfbterms.py index bf715d9..66951f1 100644 --- a/vfbterms.py +++ b/vfbterms.py @@ -9,6 +9,8 @@ import json import traceback import time +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED from urllib.parse import quote # Suppress the urllib3 warning about OpenSSL @@ -25,17 +27,24 @@ from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry -version = 8 +version = 9 API_BASE = "https://v3-cached.virtualflybrain.org/get_term_info" STATUS_URL = "https://vfbquery.virtualflybrain.org/status" VFB_BROWSER_BASE = "https://v2.virtualflybrain.org/org.geppetto.frontend/geppetto" # Throttle settings — stay under 20 concurrent to keep API reliable -MAX_ACTIVE_BEFORE_BACKOFF = 20 # Back off when this many queries are active +MAX_ACTIVE_BEFORE_BACKOFF = 20 # Back off when the server reports this many active queries STATUS_CHECK_INTERVAL = 10 # Seconds between status checks while waiting MAX_CAPACITY_WAIT_SECONDS = 600 # Give up waiting for capacity after this long -API_TIMEOUT_SECONDS = int(os.environ.get("VFB_API_TIMEOUT_SECONDS", "9000")) +# Concurrent fetch/generate workers. Kept below MAX_ACTIVE_BEFORE_BACKOFF so our +# own load leaves headroom for other clients; the serial loop only ever used one +# slot, leaving the server (≈20 slots) mostly idle — this is the main speed-up. +MAX_WORKERS = int(os.environ.get("VFB_MAX_WORKERS", "12")) +# Per-request timeout. The old 9000s default was a workaround for serial stalls; +# with concurrency a hung term must not pin a worker for hours. Override with +# VFB_API_TIMEOUT_SECONDS if a genuinely heavy term needs longer. +API_TIMEOUT_SECONDS = int(os.environ.get("VFB_API_TIMEOUT_SECONDS", "600")) # Known ID prefixes for internal link conversion KNOWN_PREFIXES = ( @@ -52,11 +61,12 @@ def create_session(): """Create a requests session with retry logic and connection pooling.""" session = requests.Session() retry = Retry( - total=2, - backoff_factor=120, - status_forcelist=[500, 502, 503, 504], + total=3, + backoff_factor=5, + status_forcelist=[429, 500, 502, 503, 504], + respect_retry_after_header=True, ) - adapter = HTTPAdapter(max_retries=retry, pool_connections=10, pool_maxsize=10) + adapter = HTTPAdapter(max_retries=retry, pool_connections=MAX_WORKERS, pool_maxsize=MAX_WORKERS) session.mount("https://", adapter) session.mount("http://", adapter) return session @@ -78,6 +88,37 @@ def check_server_status(): print(f"WARNING: Could not check server status: {e}") return None +# Cached, thread-safe view of server capacity so a pool of workers doesn't each +# poll /status before every request. One worker refreshes the cache at most once +# per STATUS_CHECK_INTERVAL; the rest read the cached verdict. +_status_lock = threading.Lock() +_status_cache = {"ts": 0.0, "ok": True} + +def throttle(term_id=""): + """Pause the calling worker while the server is at/over capacity. + + Reads a shared cached /status verdict (refreshed at most once per + STATUS_CHECK_INTERVAL). Returns once the server has headroom, or after + MAX_CAPACITY_WAIT_SECONDS so a single term can't block forever. With + MAX_WORKERS below MAX_ACTIVE_BEFORE_BACKOFF our own load never trips this; + it only engages when other clients are loading the server. + """ + start = time.time() + while True: + now = time.time() + with _status_lock: + if (now - _status_cache["ts"]) >= STATUS_CHECK_INTERVAL: + st = check_server_status() + _status_cache["ts"] = now + _status_cache["ok"] = True if st is None else st[0] < MAX_ACTIVE_BEFORE_BACKOFF + ok = _status_cache["ok"] + if ok: + return + if time.time() - start >= MAX_CAPACITY_WAIT_SECONDS: + print(f"WARNING: proceeding with {term_id or 'request'} after waiting for capacity") + return + time.sleep(STATUS_CHECK_INTERVAL) + def wait_for_server_capacity(term_id=""): """Block until the server has capacity below our threshold. @@ -131,8 +172,8 @@ def fetch_term_info(term_id): Checks server capacity before making the request to avoid flooding. """ - # Wait until the server isn't overloaded - wait_for_server_capacity(term_id) + # Wait until the server isn't overloaded (shared cached capacity check) + throttle(term_id) try: resp = session.get(API_BASE, params={"id": term_id}, timeout=API_TIMEOUT_SECONDS) @@ -192,6 +233,32 @@ def replace_link(match): # Match: [ ... ]( ID ) where the ID part has no spaces or parens return re.sub(r'\[([^\]]*(?:\[[^\]]*\][^\]]*)*)\]\(([^)\s]+)\)', replace_link, text) +# Markdown links nested inside a raw HTML block (e.g. the hero card

) are NOT +# processed by Hugo's Goldmark renderer — it passes HTML blocks through verbatim. +# So any `[label](id)` placed inside

…

would render as literal brackets. +# linkify_html emits real anchors instead, mapping known ids to /reports/ +# exactly like convert_internal_links does for plain markdown. +_MD_LINK_RE = re.compile(r'\[([^\]]*(?:\[[^\]]*\][^\]]*)*)\]\(([^)\s]+)\)') + +def linkify_html(text): + """Convert `[label](target)` markdown links to HTML anchors. + + For embedding in raw HTML blocks (hero card description/comment), where + Goldmark won't process markdown. Known VFB ids are rewritten to /reports/; + other targets (already-resolved URLs) are used as-is. Newlines are collapsed + to spaces so the surrounding HTML block isn't terminated by a blank line. + """ + if not text: + return "" + + def repl(match): + label = match.group(1) + identifier = match.group(2) + href = get_report_url(identifier) if is_known_id(identifier) else identifier + return f'{label}' + + return _MD_LINK_RE.sub(repl, text).replace("\r", " ").replace("\n", " ") + def format_relationships_section(relationships_text): """Format Meta.Relationships into markdown bullets. @@ -305,7 +372,16 @@ def format_synonyms_table(synonyms): return "\n".join(lines) def format_query_preview(query, term_id): - """Format a Query's preview_results as a markdown table with thumbnails.""" + """Format a Query's preview_results as an HTML table with thumbnails. + + Emitted as a complete raw HTML rather than a markdown table wrapped + in
. Hugo's Goldmark does not process markdown + nested inside a raw HTML block, so the wrapped-markdown form leaked every + `[label](id)` cell as literal text and never rendered as a table. A full + HTML table renders verbatim under unsafe=true. The `###` heading and the + "View all" button stay outside the block, separated by blank lines so + Goldmark closes each HTML block cleanly. + """ preview = query.get("preview_results", {}) rows = preview.get("rows", []) if not rows: @@ -319,36 +395,39 @@ def format_query_preview(query, term_id): lines.append(f'### {label} ({count} total)') lines.append("") lines.append('
') - lines.append("") - lines.append("| Thumbnail | Name | Tags |") - lines.append("|-----------|------|------|") + lines.append('
') + lines.append('') + lines.append('') for row in rows: - # Extract name — may be markdown link like [name](id) + # Name — may be `[label](id)` markdown from the API; emit as an anchor + # because it lives inside a raw HTML block (Goldmark won't linkify it). name_raw = row.get("label", row.get("name", "")) - name_converted = convert_internal_links(name_raw) + name_md = convert_internal_links(name_raw) row_id = row.get("id", "") - if row_id and is_known_id(row_id) and name_converted == name_raw and name_raw: - name_converted = f'[{name_raw}]({get_report_url(row_id)})' + if row_id and is_known_id(row_id) and name_md == name_raw and name_raw: + name_md = f'[{name_raw}]({get_report_url(row_id)})' + name_html = linkify_html(name_md) if "](" in name_md else name_md - # Extract tags + # Tags tags_raw = row.get("tags", "") tags_display = tags_raw.replace("|", ", ") if tags_raw else "" - # Extract thumbnail — may be markdown image like [![alt](url)](link) + # Thumbnail — pull the URL out of `[![alt](url 'title')](link)` markdown. thumb_raw = row.get("thumbnail", "") thumb_html = "" if thumb_raw: - # Extract URL from markdown image: [![alt](url "title")](link) img_match = re.search(r'!\[[^\]]*\]\(([^\s)]+)', thumb_raw) if img_match: thumb_url = img_match.group(1) - thumb_html = f'' + thumb_html = (f'' + f'') - lines.append(f'| {thumb_html} | {name_converted} | {tags_display} |') + lines.append(f'') - lines.append("") - lines.append("") + lines.append('') + lines.append('
ThumbnailNameTags
{thumb_html}{name_html}{tags_display}
') + lines.append('') lines.append("") lines.append(f'View all {count} results in VFB →') lines.append("") @@ -427,6 +506,48 @@ def format_publications(publications): lines.append("".join(parts)) return "\n".join(lines) +def format_xrefs(xrefs, term_name): + """Format Xrefs (external cross-references) into markdown links. + + Each xref is {label, accession, link, icon}. The cached get_term_info + endpoint resolves the full external URL in `link`, so we link to it + directly rather than rebuilding from a link_base/accession template + (cf. VFBProcessTermInfoJson.xref.getLink). Link text mirrors the VFB + browser's "Cross References" section: " on ". + """ + if not xrefs: + return "" + + lines = [] + for xref in xrefs: + link = xref.get("link", "") + if not link: + continue + label = xref.get("label", "") + text = f'{term_name} on {label}' if (term_name and label) else (label or link) + accession = xref.get("accession", "") + suffix = f' ({accession})' if accession and accession not in ("None", "") else "" + lines.append(f'- [{text}]({link}){suffix}') + return "\n".join(lines) + +def format_related_tools(tools, term_id): + """Format RelatedTools into the VFB browser's "Graphs For" links. + + Each tool is {tool, label, default_args}. These launch graph/hierarchy + tools on the term inside the 3D browser; link to the term there so the + user can run them (the browser exposes the tool set once the term loads). + """ + if not tools: + return "" + + lines = [] + for t in tools: + label = t.get("label", t.get("tool", "")) + if not label: + continue + lines.append(f'- [{label}]({VFB_BROWSER_BASE}?id={term_id})') + return "\n".join(lines) + def build_hero_card(name, term_id, tags_badges, description_html, comment_html, thumbnails): """Build the hero card without blank lines that break Markdown HTML blocks.""" lines = [ @@ -480,13 +601,19 @@ def generate_page(term_data): publications = term_data.get("Publications", []) technique = term_data.get("Technique", []) images = term_data.get("Images", {}) + examples = term_data.get("Examples", {}) + xrefs = term_data.get("Xrefs", []) + related_tools = term_data.get("RelatedTools", []) # Description for front matter description = meta.get("Description", "") if not description: - # Fallback to Types text (stripped of markdown links) - types_text = meta.get("Types", "") - description = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', types_text) + # Fallback to Types text + description = meta.get("Types", "") + # Strip markdown link syntax to plain labels: the front-matter description + # becomes the tag and page summary, where raw + # `[label](id)` syntax would show literally. + description = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', description) # Clean for YAML description = description.replace('"', '\\"').replace("\n", " ").replace("\r", " ").strip() @@ -497,11 +624,14 @@ def generate_page(term_data): url_slug = get_term_url(name, term_id) # Tags for front matter - tags_csv = ",".join(tags) + # Build a clean list and drop empties, otherwise a term with no Tags + # yields a leading comma (e.g. "[,VFB]") which is invalid YAML flow syntax + tag_list = list(tags) if "_" in term_id: - tags_csv += "," + term_id.split("_")[0] + tag_list.append(term_id.split("_")[0]) elif term_id.startswith("FB"): - tags_csv += "," + term_id[0:4] + tag_list.append(term_id[0:4]) + tags_csv = ",".join(t for t in tag_list if t) # Thumbnails thumbnails = get_thumbnails(term_data) @@ -529,11 +659,11 @@ def generate_page(term_data): desc_html = "" if meta.get("Description"): - desc_html = f'

{convert_internal_links(meta["Description"])}

' + desc_html = f'

{linkify_html(meta["Description"])}

' comment_html = "" if comment: - comment_html = f'

{convert_internal_links(comment)}

' + comment_html = f'

{linkify_html(comment)}

' sections.append(build_hero_card(name, term_id, tags_badges, desc_html, comment_html, thumbnails)) @@ -554,6 +684,12 @@ def generate_page(term_data): syn_md = format_synonyms_table(synonyms) sections.append(f'## Alternative Names\n\n{syn_md}\n') + # ── Cross references (external databases) ── + if xrefs: + xref_md = format_xrefs(xrefs, name) + if xref_md.strip(): + sections.append(f'## Cross References\n\n{xref_md}\n') + # ── Technique ── if technique: tech_lines = "\n".join(f'- {t}' for t in technique) @@ -564,9 +700,17 @@ def generate_page(term_data): lic_md = format_licenses(licenses) sections.append(f'## License\n\n{lic_md}\n') - # ── Downloads ── - if images: - dl_md = format_downloads(images) + # ── Downloads (individual Images and class Examples) ── + # Class terms carry their downloadable volumes under Examples (one list + # per template), individuals under Images. Merge both so class pages get + # download links too, matching the browser's Available Images downloads. + download_sources = {} + for tpl, img_list in images.items(): + download_sources.setdefault(tpl, []).extend(img_list) + for tpl, ex_list in examples.items(): + download_sources.setdefault(tpl, []).extend(ex_list) + if download_sources: + dl_md = format_downloads(download_sources) if dl_md.strip(): sections.append(f'## Downloads\n\n{dl_md}\n') @@ -578,12 +722,22 @@ def generate_page(term_data): q_md = format_query_preview(q, term_id) sections.append(q_md) + # ── Graphs For (related graph/hierarchy tools) ── + if related_tools: + tools_md = format_related_tools(related_tools, term_id) + if tools_md.strip(): + sections.append(f'## Graphs For\n\n{tools_md}\n') + # ── Publications ── if publications: pub_md = format_publications(publications) sections.append(f'## References\n\n{pub_md}\n') - return "\n".join(sections) + # Join with a blank line between every section. Goldmark keeps a raw HTML + # block (e.g. the hero card) open until a blank line, so without this the + # first heading after the hero ("## Classification") gets swallowed into + # the HTML block and printed literally instead of rendered as a heading. + return "\n\n".join(sections) # ─── Term Saving ───────────────────────────────────────────────────────────── @@ -610,46 +764,79 @@ def process_group(base_path, relative_dir, label, query): chdir(target_dir) save_terms(fetch_ids(label, query)) -def save_terms(ids): - """Fetch and save term pages for a list of IDs.""" - total = len(ids) - success_count = 0 - skip_count = 0 - fail_count = 0 - for i, term_id in enumerate(ids): - try: - filename = term_id + "_v" + str(version) + ".md" - if os.path.isfile(filename): - skip_count += 1 - continue +def process_term(term_id): + """Fetch, render and write one term page. Returns a status string. - print(f"Processing {term_id} ({i+1}/{total})...") - term_data = fetch_term_info(term_id) - if term_data is None: - fail_count += 1 - continue + Runs inside a worker thread. Writes to a temp file then atomically renames + so a killed build never leaves a half-written page. Throttling against the + server's capacity happens inside fetch_term_info via throttle(). + """ + filename = term_id + "_v" + str(version) + ".md" + if os.path.isfile(filename): + return "skip" - page_content = generate_page(term_data) + term_data = fetch_term_info(term_id) + if term_data is None: + return "fail" - with open(filename, "w", encoding="utf-8") as f: - f.write(page_content) + page_content = generate_page(term_data) - success_count += 1 + tmp = f"{filename}.{os.getpid()}.{threading.get_ident()}.tmp" + with open(tmp, "w", encoding="utf-8") as f: + f.write(page_content) + os.replace(tmp, filename) - # Clean up previous version - old_filename = term_id + "_v" + str(version - 1) + ".md" - if os.path.isfile(old_filename): - os.remove(old_filename) - print(f'Removed: {old_filename}') + # Clean up previous version + old_filename = term_id + "_v" + str(version - 1) + ".md" + if os.path.isfile(old_filename): + try: + os.remove(old_filename) + except OSError: + pass + return "ok" - # Throttling is handled by wait_for_server_capacity() in fetch_term_info +def save_terms(ids): + """Fetch and save term pages for a list of IDs, concurrently. + + A bounded ThreadPoolExecutor keeps up to MAX_WORKERS requests in flight so + the server's ~20 query slots stay busy, instead of the old one-at-a-time + loop that left them idle. The in-flight window is capped so building the + future set for a 600k-id group doesn't balloon memory. + """ + total = len(ids) + counts = {"ok": 0, "skip": 0, "fail": 0} + done = 0 + window = MAX_WORKERS * 4 + progress_every = max(100, window) + def record(fut, tid): + nonlocal done + try: + status = fut.result() except Exception as e: - fail_count += 1 - print(f"ERROR processing {term_id}: {str(e)}") + status = "fail" + print(f"ERROR processing {tid}: {e}") print(traceback.format_exc()) - - print(f"\nBatch complete: {success_count} created, {skip_count} skipped (existing), {fail_count} failed out of {total} total") + counts[status] = counts.get(status, 0) + 1 + done += 1 + if status == "fail" or done % progress_every == 0: + print(f" {done}/{total} processed " + f"(created={counts['ok']} skipped={counts['skip']} failed={counts['fail']}) " + f"last={tid}:{status}") + + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex: + inflight = {} + for term_id in ids: + inflight[ex.submit(process_term, term_id)] = term_id + if len(inflight) >= window: + finished, _ = wait(set(inflight), return_when=FIRST_COMPLETED) + for fut in finished: + record(fut, inflight.pop(fut)) + for fut in as_completed(set(inflight)): + record(fut, inflight[fut]) + + print(f"\nBatch complete: {counts['ok']} created, {counts['skip']} skipped (existing), " + f"{counts['fail']} failed out of {total} total") # ─── Testing ───────────────────────────────────────────────────────────────── @@ -695,12 +882,16 @@ def test_term_page(term_id, term_type="class"): checks["Relationships"] = "## Relationships" in page_content if term_data.get("Synonyms"): checks["Synonyms"] = "## Alternative Names" in page_content + if term_data.get("Xrefs"): + checks["Cross References"] = "## Cross References" in page_content if term_data.get("Licenses"): checks["Licenses"] = "## License" in page_content - if term_data.get("Images"): + if term_data.get("Images") or term_data.get("Examples"): checks["Downloads"] = "## Downloads" in page_content if any(q.get("preview_results", {}).get("rows") for q in term_data.get("Queries", [])): checks["Query previews"] = "table-responsive" in page_content + if term_data.get("RelatedTools"): + checks["Graphs For"] = "## Graphs For" in page_content all_pass = True for check_name, result in checks.items(): @@ -788,8 +979,8 @@ def test_report_link_regression(): checks = { "Markdown links use report path": converted == "[DNb08](/reports/FBbt_20011340)", - "Query preview preserves report link": "[DNb08](/reports/FBbt_20011340)" in query_preview, - "Plain row labels get report link": "[DNp45](/reports/FBbt_20011346)" in query_preview, + "Query preview preserves report link": 'DNb08' in query_preview, + "Plain row labels get report link": 'DNp45' in query_preview, } all_pass = True