diff --git a/.gitignore b/.gitignore index 9f173f5dd5..36e26714d6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ node_modules .astro/ dist +# Local Cloudflare Pages runtime (`wrangler pages dev`) build artifacts +.wrangler/ + # Statically generated images at build time, don't push them /static/og-images diff --git a/biome.json b/biome.json index 770b203402..7ed12a281c 100644 --- a/biome.json +++ b/biome.json @@ -4,6 +4,7 @@ "includes": [ "**/*", "!.astro/*", + "!.wrangler/**", "!src/@types/*", "!public/api-schemas.json", "!public/cli-schema.json", diff --git a/eslint.config.js b/eslint.config.js index 894a8bf48c..934be413d4 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -14,6 +14,7 @@ export default [ ignores: [ 'dist/**', '.astro/**', + '.wrangler/**', 'node_modules/**', '.github/**', '.claude/**', diff --git a/functions/_middleware.test.ts b/functions/_middleware.test.ts new file mode 100644 index 0000000000..4cc1211bc3 --- /dev/null +++ b/functions/_middleware.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from 'vitest'; +import { onRequest } from './_middleware'; + +/** + * Stands in for Cloudflare's asset server. `assets` maps a pathname to the + * response it would serve; anything absent 404s, as it would in production. + * + * `next()` with no argument re-runs the original request, which is how the + * middleware asks for the HTML after the Markdown twin came back missing. + */ +async function run( + path: string, + { + assets, + accept, + method = 'GET', + }: { assets: Record; accept?: string; method?: string } +): Promise<{ response: Response; seen: string[] }> { + const request = new Request(`https://docs.mergify.com${path}`, { + method, + headers: accept ? { Accept: accept } : {}, + }); + const seen: string[] = []; + const next = async (input: Request) => { + // The middleware must always name the request it wants. A bare `next()` is + // documented as re-forwarding the original, but the Pages runtime forwards + // the last request it was given instead, so relying on it served the + // Markdown 404 for pages that only have an HTML twin. + if (!input) throw new Error('next() was called without an explicit Request'); + const pathname = new URL(input.url).pathname; + seen.push(pathname); + const asset = assets[pathname]; + // Pages serves the built `404.html` for anything it does not have. + return ( + asset?.clone() ?? + new Response('not found', { + status: 404, + headers: { 'Content-Type': 'text/html; charset=utf-8' }, + }) + ); + }; + return { response: await onRequest({ request, next }), seen }; +} + +const html = (body = '', status = 200) => + new Response(body, { status, headers: { 'Content-Type': 'text/html; charset=utf-8' } }); +const markdown = (body = '# Merge Queue') => + new Response(body, { headers: { 'Content-Type': 'text/markdown; charset=utf-8' } }); + +describe('markdown content negotiation', () => { + it('serves the Markdown twin when Markdown is asked for', async () => { + const { response, seen } = await run('/merge-queue', { + accept: 'text/markdown', + assets: { '/merge-queue.md': markdown(), '/merge-queue': html() }, + }); + + expect(await response.text()).toBe('# Merge Queue'); + expect(response.headers.get('vary')).toBe('Accept'); + expect(seen).toEqual(['/merge-queue.md']); + }); + + it('passes a 304 straight through instead of falling back to HTML', async () => { + // Regression: `.ok` is false for 304, so a client revalidating Markdown it + // already holds was answered with the HTML page. + const { response, seen } = await run('/merge-queue', { + accept: 'text/markdown', + assets: { '/merge-queue.md': new Response(null, { status: 304 }), '/merge-queue': html() }, + }); + + expect(response.status).toBe(304); + expect(response.headers.get('vary')).toBe('Accept'); + expect(seen).toEqual(['/merge-queue.md']); + }); + + it('falls back to HTML for a page with no Markdown twin', async () => { + // `/api` and `/cli` are built from `src/pages/`, so no `.md` is generated. + const { response, seen } = await run('/api', { + accept: 'text/markdown', + assets: { '/api': html('api') }, + }); + + expect(response.status).toBe(200); + expect(await response.text()).toBe('api'); + // The second call must ask for the original path, not repeat the `.md` one. + expect(seen).toEqual(['/api.md', '/api']); + }); + + it('answers a missing page in the format that was asked for', async () => { + const { response } = await run('/nope', { accept: 'text/markdown', assets: {} }); + + expect(response.status).toBe(404); + expect(response.headers.get('content-type')).toContain('text/markdown'); + expect(await response.text()).toContain('llms.txt'); + + const { response: asHtml } = await run('/nope', { assets: {} }); + expect(asHtml.status).toBe(404); + expect(asHtml.headers.get('content-type')).toContain('text/html'); + }); +}); + +describe('everything else', () => { + it('serves HTML to a browser, and marks the response as negotiated', async () => { + const { response, seen } = await run('/merge-queue', { + accept: 'text/html,application/xhtml+xml,*/*;q=0.8', + assets: { '/merge-queue': html() }, + }); + + expect(response.headers.get('content-type')).toContain('text/html'); + expect(response.headers.get('vary')).toBe('Accept'); + expect(seen).toEqual(['/merge-queue']); + }); + + it('leaves assets alone', async () => { + const { response } = await run('/_astro/app.css', { + accept: 'text/markdown', + assets: { + '/_astro/app.css': new Response('body{}', { headers: { 'Content-Type': 'text/css' } }), + }, + }); + + expect(response.headers.get('vary')).toBeNull(); + }); + + it('leaves non-GET requests alone', async () => { + const { response, seen } = await run('/merge-queue', { + accept: 'text/markdown', + method: 'POST', + assets: { '/merge-queue': html() }, + }); + + expect(response.headers.get('vary')).toBeNull(); + expect(seen).toEqual(['/merge-queue']); + }); + + it('preserves a Vary the asset server already set', async () => { + const { response } = await run('/merge-queue', { + assets: { + '/merge-queue': new Response('', { + headers: { 'Content-Type': 'text/html', Vary: 'Accept-Encoding' }, + }), + }, + }); + + expect(response.headers.get('vary')).toBe('Accept-Encoding, Accept'); + }); +}); diff --git a/functions/_middleware.ts b/functions/_middleware.ts new file mode 100644 index 0000000000..a31225fc85 --- /dev/null +++ b/functions/_middleware.ts @@ -0,0 +1,112 @@ +import { isNegotiablePage, prefersMarkdown } from '../src/util/acceptMarkdown'; +import { getMarkdownPath } from '../src/util/getMarkdownPath'; + +/** + * Cloudflare Pages middleware: serve the Markdown twin of a page to clients that + * ask for it with `Accept: text/markdown` (https://acceptmarkdown.com). + * + * The site is a static build, so this is the only layer that sees request + * headers. It stays deliberately thin — the decision itself lives in + * `src/util/acceptMarkdown.ts`, where it is unit-tested. + */ + +/** + * The slice of Cloudflare's `EventContext` we use. Declared structurally rather + * than pulling in `@cloudflare/workers-types`: this is the only Worker in the + * repo, and `tsconfig.json` scopes typechecking to `src/`. + */ +interface MiddlewareContext { + request: Request; + /** + * Always call this with an explicit request. A bare `next()` is documented as + * forwarding the original one, but once the middleware has asked for the + * Markdown twin the runtime forwards *that* request again instead — which + * answered `/api`, a page with no twin, with the Markdown 404 rather than its + * HTML. + */ + next: (input: Request) => Promise; +} + +/** Body served when an agent asks for Markdown and the path does not exist. */ +const MARKDOWN_404 = `# 404 — Page not found + +This path does not exist in the Mergify documentation. + +- [Documentation index](https://docs.mergify.com/index.md) +- [llms.txt](https://docs.mergify.com/llms.txt) — every page, with descriptions +- [Sitemap](https://docs.mergify.com/sitemap-index.xml) +- [OpenAPI description](https://docs.mergify.com/openapi.json) — the Mergify REST API + +Most documentation pages also serve their Markdown source: append \`.md\` to the +URL, or send \`Accept: text/markdown\`. The generated API and CLI references are +HTML only. +`; + +/** + * Republish a response with `Accept` merged into `Vary`. + * + * Without it a CDN that cached the HTML variant first would hand it to an agent + * asking for Markdown (and vice versa). Only applied to the negotiated media + * types: putting `Vary: Accept` on images and CSS would fragment their cache + * keys for nothing. + */ +function withVaryAccept(response: Response): Response { + const contentType = response.headers.get('content-type') ?? ''; + const negotiated = + contentType.startsWith('text/html') || + contentType.startsWith('text/markdown') || + // A `304 Not Modified` carries no content type but still answers for one + // of the two variants, and needs the header most of all: it is the reply + // to a cache that is about to reuse a stored representation. + response.status === 304; + if (!negotiated) return response; + + const varied = new Response(response.body, response); + const existing = varied.headers.get('vary'); + const fields = new Set( + (existing ?? '') + .split(',') + .map((field) => field.trim()) + .filter(Boolean) + ); + fields.add('Accept'); + varied.headers.set('Vary', Array.from(fields).join(', ')); + return varied; +} + +export async function onRequest(context: MiddlewareContext): Promise { + const { request, next } = context; + + if (request.method !== 'GET' && request.method !== 'HEAD') return next(request); + + const url = new URL(request.url); + if (!isNegotiablePage(url.pathname)) return next(request); + + if (!prefersMarkdown(request.headers.get('accept'))) { + return withVaryAccept(await next(request)); + } + + const markdownUrl = new URL(getMarkdownPath(url.pathname), url); + markdownUrl.search = url.search; + const markdown = await next(new Request(markdownUrl, request)); + + // Fall back to HTML only when there is genuinely no Markdown twin. Keying + // this off `ok` would also catch `304 Not Modified` — the normal answer to a + // client revalidating Markdown it already holds — and serve it HTML instead, + // and would turn a redirect or a 5xx from the `.md` route into HTML too. + if (markdown.status !== 404) return withVaryAccept(markdown); + + // No Markdown twin: either a page we do not generate one for (`/api` and + // `/cli` are built from `src/pages/`), or a path that does not exist at all. + // Answer 404s in the format that was asked for; hand anything else back as + // HTML. + const html = await next(request); + if (html.status !== 404) return withVaryAccept(html); + + return withVaryAccept( + new Response(MARKDOWN_404, { + status: 404, + headers: { 'Content-Type': 'text/markdown; charset=utf-8' }, + }) + ); +} diff --git a/public/_routes.json b/public/_routes.json new file mode 100644 index 0000000000..bc9b8f72dd --- /dev/null +++ b/public/_routes.json @@ -0,0 +1,23 @@ +{ + "version": 1, + "include": ["/*"], + "exclude": [ + "/_astro/*", + "/pagefind/*", + "/open-graph/*", + "/favicon.ico", + "/favicon.svg", + "/robots.txt", + "/sitemap-index.xml", + "/sitemap-0.xml", + "/llms.txt", + "/openapi.json", + "/api-schemas.json", + "/cli-schema.json", + "/mergify-configuration-schema.json", + "/make-scrollable-code-focusable.js", + "/sw.js", + "/mergify-count-contributors.py", + "/mergify-github-app-logo.png" + ] +} diff --git a/src/components/HeadSEO.astro b/src/components/HeadSEO.astro index 63c5d543eb..1d7fee1a1b 100644 --- a/src/components/HeadSEO.astro +++ b/src/components/HeadSEO.astro @@ -1,6 +1,7 @@ --- import type { CollectionEntry } from 'astro:content'; import { OPEN_GRAPH } from '../config'; +import { getMarkdownPath } from '../util/getMarkdownPath'; import { getOgImageUrl } from '../util/getOgImageUrl'; export interface Props { @@ -9,6 +10,7 @@ export interface Props { } const { content, canonicalURL } = Astro.props; +const markdownURL = new URL(getMarkdownPath(canonicalURL.pathname), canonicalURL); const ogImageUrl = getOgImageUrl(canonicalURL.pathname); const imageSrc = ogImageUrl; const canonicalImageSrc = imageSrc ? new URL(imageSrc, Astro.site) : undefined; @@ -20,6 +22,9 @@ const siteDescription = + + diff --git a/src/util/acceptMarkdown.test.ts b/src/util/acceptMarkdown.test.ts new file mode 100644 index 0000000000..cae8d27157 --- /dev/null +++ b/src/util/acceptMarkdown.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest'; +import { isNegotiablePage, prefersMarkdown } from './acceptMarkdown'; + +describe('prefersMarkdown', () => { + it('serves Markdown when it is asked for explicitly', () => { + expect(prefersMarkdown('text/markdown')).toBe(true); + expect(prefersMarkdown('text/markdown, text/plain;q=0.5')).toBe(true); + expect(prefersMarkdown('TEXT/MARKDOWN')).toBe(true); + }); + + it('leaves browsers alone', () => { + // Chrome and Firefox both rank HTML first and end on a `*/*` catch-all. If + // wildcards counted, every human visitor would be served raw Markdown. + expect( + prefersMarkdown( + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' + ) + ).toBe(false); + expect(prefersMarkdown('*/*')).toBe(false); + expect(prefersMarkdown('text/*')).toBe(false); + }); + + it('respects q-values when both formats are named', () => { + expect(prefersMarkdown('text/markdown;q=0.9, text/html;q=0.8')).toBe(true); + expect(prefersMarkdown('text/markdown;q=0.5, text/html')).toBe(false); + expect(prefersMarkdown('text/markdown;q=0, text/html;q=0')).toBe(false); + }); + + it('falls back to HTML when nothing was asked for', () => { + expect(prefersMarkdown(null)).toBe(false); + expect(prefersMarkdown('')).toBe(false); + }); +}); + +describe('isNegotiablePage', () => { + it('matches documentation pages', () => { + expect(isNegotiablePage('/')).toBe(true); + expect(isNegotiablePage('/merge-queue')).toBe(true); + expect(isNegotiablePage('/api/usage/')).toBe(true); + }); + + it('leaves assets untouched', () => { + expect(isNegotiablePage('/merge-queue.md')).toBe(false); + expect(isNegotiablePage('/openapi.json')).toBe(false); + expect(isNegotiablePage('/llms.txt')).toBe(false); + expect(isNegotiablePage('/_astro/MainLayout.lvKlb881.css')).toBe(false); + expect(isNegotiablePage('/open-graph/index.png')).toBe(false); + }); +}); diff --git a/src/util/acceptMarkdown.ts b/src/util/acceptMarkdown.ts new file mode 100644 index 0000000000..97902afb92 --- /dev/null +++ b/src/util/acceptMarkdown.ts @@ -0,0 +1,66 @@ +/** + * Content negotiation for the Markdown twin of every docs page. + * + * Every page already ships a `.md` source at `.md` (see + * `src/pages/[...slug].md.ts`), which is what the "View as Markdown" button and + * `llms.txt` link to. The convention agents actually try first, though, is to + * ask for the page's own URL with `Accept: text/markdown` — see + * https://acceptmarkdown.com. This module decides when to honour that. + */ + +interface AcceptEntry { + type: string; + q: number; +} + +function parseAccept(header: string): AcceptEntry[] { + return header + .split(',') + .map((part) => { + const [rawType, ...params] = part.split(';'); + const type = rawType.trim().toLowerCase(); + if (!type) return undefined; + // Only `q` matters to us; any other accept-param is ignored. + let q = 1; + for (const param of params) { + const [key, value] = param.split('='); + if (key?.trim().toLowerCase() !== 'q') continue; + const parsed = Number.parseFloat(value ?? ''); + if (Number.isFinite(parsed)) q = parsed; + } + return { type, q }; + }) + .filter((entry): entry is AcceptEntry => entry !== undefined); +} + +/** + * Whether a request asking for `Accept:
` should be served Markdown. + * + * Deliberately requires `text/markdown` to be named explicitly: browsers send + * `text/html,...,*\/*;q=0.8`, so honouring wildcards would hand Markdown to + * every human visitor whose browser happens to list HTML at a lower q than the + * catch-all. An agent that wants Markdown says so. + */ +export function prefersMarkdown(header: string | null | undefined): boolean { + if (!header) return false; + + const entries = parseAccept(header); + const markdown = entries.find((entry) => entry.type === 'text/markdown'); + if (!markdown || markdown.q <= 0) return false; + + // A client that lists both and ranks HTML higher gets HTML. + const html = entries.find((entry) => entry.type === 'text/html'); + return html === undefined || markdown.q >= html.q; +} + +/** + * Whether a path is a docs page with a Markdown twin, as opposed to an asset. + * + * Pages are extensionless (`/merge-queue`, `/api/usage/`); anything carrying a + * file extension is a static asset and is served untouched. + */ +export function isNegotiablePage(pathname: string): boolean { + if (pathname === '/') return true; + const lastSegment = pathname.replace(/\/$/, '').split('/').pop() ?? ''; + return !lastSegment.includes('.'); +}