docs(site): generate the llms.txt corpus from the built site

.github/workflows/docs-site-build.yml

@@ -41,12 +41,6 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Verify llms.txt artifacts are up to date
-        if: steps.guard.outputs.present == 'true'
-        run: |
-          python3 docs/site/scripts/generate-llms.py --check
-          python3 -m unittest discover docs/site/scripts -v
-
       - name: Verify disk-size markers are in sync
         if: steps.guard.outputs.present == 'true'
         run: python3 docs/site/scripts/render-disk-sizes.py --check
@@ -115,3 +109,14 @@ jobs:
           # Version resolution fails the build rather than publishing a
           # placeholder, so it must not run against the anonymous rate limit.
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # Both require docs/site/build, so they have to follow the build rather
+      # than precede it: the built-site tests raise a setup error without it,
+      # and the artifact check regenerates from the built pages.
+      - name: Run documentation script tests
+        if: steps.guard.outputs.present == 'true'
+        run: python3 -m unittest discover docs/site/scripts -v
+
+      - name: Verify llms.txt artifacts are up to date
+        if: steps.guard.outputs.present == 'true'
+        run: python3 docs/site/scripts/generate-llms.py --check

.github/workflows/docs-version-bump.yml

@@ -123,16 +123,18 @@ jobs:
             jq '.[:5]' versions.json > versions.json.tmp && mv versions.json.tmp versions.json
           fi
 
-      - name: Regenerate llms.txt artifacts
-        run: python3 scripts/generate-llms.py
-
       - name: Verify build
         run: npm run build
         env:
           # Same reasoning as the snapshot step: `npm run build` loads the same
           # branch-controlled config, so it gets the read-only token too.
           GITHUB_TOKEN: ${{ steps.docs_token.outputs.token }}
 
+      # Page bodies are read from the built site, so this has to follow the
+      # build: running it first regenerates the corpus from the previous one.
+      - name: Regenerate llms.txt artifacts
+        run: python3 scripts/generate-llms.py
+
       - name: Open pull request
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

docs/site/docs/fundamentals/mcp.mdx

@@ -337,9 +337,9 @@ MCP connects a model to your **running node**. To let a model answer questions a
 | [llms.txt](https://docs.erigon.tech/llms.txt) | Index of the current documentation, with titles and descriptions | The model fetches pages on demand, or its context window is small |
 | [llms-full.txt](https://docs.erigon.tech/llms-full.txt) | The text of those same pages, as one plain-text file | You want the whole corpus in context, offline, with nothing scraped |
 
-Both are generated from the documentation sources by a script in the repository, and CI fails when the committed files differ from what that script produces. The check compares the artifacts against the generator's own output, so it does not prove that the generator carried every part of a page across.
+Both are generated from this site's built pages by a script in the repository, and CI fails when the committed files differ from what that script produces — except for a difference confined to the Erigon release token, which is reported as a warning so that a new release does not break every open documentation pull request. The check compares the artifacts against the generator's own output, so it does not prove that the generator carried every part of a page across.
 
-Two things to know about their scope. They cover the **current** documentation and the help center only — archived versions, published under `/vX.Y/`, are not included. And the text is cleaned rather than copied verbatim: MDX components are stripped, and section index pages built as card grids are reduced to the list of links they present, so prose outside the cards on those pages is not carried. Read the corpus as the documentation's prose, not as a byte-for-byte mirror of the rendered site.
+Two things to know about their scope. They cover the **current** documentation and the help center only — archived versions, published under `/vX.Y/`, are not included. And the text is cleaned rather than copied verbatim: components are expanded and links resolved as the site renders them, and a section index page built as a card grid keeps its prose with each grid rendered as the list of links it presents. Read the corpus as the documentation's prose, not as a byte-for-byte mirror of the rendered site.
 
 ```bash
 # Give a local model the whole documentation, with no network access

docs/site/docusaurus.config.ts

@@ -76,8 +76,8 @@ export default async function createConfig(): Promise<Config> {
     markdown: {mermaid: true},
     themes: ['@docusaurus/theme-mermaid'],
 
-    // Both lists reach src/theme/Root.tsx, which uses them to keep the llms.txt
-    // descriptor off the routes the index does not cover.
+    // archivedVersions reaches src/theme/Root.tsx, which uses it to keep the
+    // llms.txt descriptor off versions the current index does not cover.
     customFields: {latestVersion, archivedVersions},
 
     headTags: [

docs/site/package.json

@@ -33,6 +33,9 @@
     "@docusaurus/tsconfig": "3.10.2",
     "@docusaurus/types": "3.10.2",
     "@types/react": "^19.2.18",
+    "mdast-util-from-markdown": "2.0.3",
+    "mdast-util-gfm": "3.1.0",
+    "micromark-extension-gfm": "3.0.0",
     "typescript": "6.0.3"
   },
   "browserslist": {