Skip to content

Commit ba28a20

Browse files
prc5claude
andcommitted
feat: 🎸 /sitemap.xml and /robots.txt generated from the frontend route tree
Both are TanStack Start server routes. The sitemap is built from router.routesByPath, so adding a page file adds an entry - dynamic routes, layouts and server-only routes drop out on their own, and a page opts out with `staticData: { sitemap: false }`. The origin comes from the request (x-forwarded-* aware), so no environment needs configuring, and robots.txt only invites crawlers when VITE_ENVIRONMENT is production. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent 387ba33 commit ba28a20

10 files changed

Lines changed: 446 additions & 3 deletions

File tree

‎apps/frontend/src/global.d.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22
/* eslint-disable import/no-default-export */
33
/// <reference types="@electron-forge/plugin-vite/forge-vite-env" />
44
/// <reference types="vite/client" />
5+
// TanStack Start is what augments route options with `server.handlers`, which
6+
// the /sitemap.xml and /robots.txt routes answer from. Nothing else in the app
7+
// pulls the package in at type level - `server.ts` imports only its entry
8+
// subpath - so the augmentation is referenced here, once, for the whole program.
9+
/// <reference types="@tanstack/react-start" />
510

611
import type { ElectronAPI } from "@electron-toolkit/preload";
712

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { buildRobotsTxt } from "./robots";
2+
3+
describe("buildRobotsTxt", () => {
4+
it("opens the site up and points at the sitemap in production", () => {
5+
// Act
6+
const robots = buildRobotsTxt("https://example.com", true);
7+
8+
// Assert
9+
expect(robots).toContain("Allow: /");
10+
expect(robots).toContain("Sitemap: https://example.com/sitemap.xml");
11+
});
12+
13+
it("blocks crawlers everywhere else so preview hosts stay unindexed", () => {
14+
// Act
15+
const robots = buildRobotsTxt("https://preview.example.com", false);
16+
17+
// Assert
18+
expect(robots).toContain("Disallow: /");
19+
expect(robots).not.toContain("Sitemap:");
20+
});
21+
});
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
export const SITEMAP_PATH = "/sitemap.xml";
2+
export const ROBOTS_PATH = "/robots.txt";
3+
4+
/**
5+
* Only the production deployment invites crawlers. Preview and staging builds
6+
* serve the same code on a public URL, and getting those indexed splits ranking
7+
* between hosts and leaks unfinished pages into search results - so everything
8+
* that is not production answers with a blanket disallow.
9+
*/
10+
export const buildRobotsTxt = (origin: string, isIndexable: boolean): string => {
11+
if (!isIndexable) {
12+
return ["User-agent: *", "Disallow: /", ""].join("\n");
13+
}
14+
15+
return ["User-agent: *", "Allow: /", "", `Sitemap: ${origin}${SITEMAP_PATH}`, ""].join("\n");
16+
};
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { getSiteOrigin } from "./site-url";
2+
3+
describe("getSiteOrigin", () => {
4+
it("uses the host the request came in on", () => {
5+
// Act
6+
const origin = getSiteOrigin(new Request("http://localhost:4200/sitemap.xml"));
7+
8+
// Assert
9+
expect(origin).toBe("http://localhost:4200");
10+
});
11+
12+
it("prefers the forwarded host and protocol behind a proxy", () => {
13+
// Arrange
14+
const request = new Request("http://10.0.0.7:8080/sitemap.xml", {
15+
headers: { "x-forwarded-host": "example.com", "x-forwarded-proto": "https" },
16+
});
17+
18+
// Act
19+
const origin = getSiteOrigin(request);
20+
21+
// Assert
22+
expect(origin).toBe("https://example.com");
23+
});
24+
25+
it("takes the first entry when several proxies appended to the chain", () => {
26+
// Arrange
27+
const request = new Request("http://10.0.0.7:8080/sitemap.xml", {
28+
headers: { "x-forwarded-host": "example.com, internal.example.com", "x-forwarded-proto": "https, http" },
29+
});
30+
31+
// Act
32+
const origin = getSiteOrigin(request);
33+
34+
// Assert
35+
expect(origin).toBe("https://example.com");
36+
});
37+
});
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/**
2+
* Absolute URLs in `robots.txt` and `sitemap.xml` have to point at the host the
3+
* crawler actually asked for, and that host is not known at build time - the same
4+
* bundle runs on localhost, on preview URLs and on the production domain. Reading
5+
* it off the request keeps every environment correct with zero configuration.
6+
*
7+
* Behind a proxy (Railway, Fly, any CDN) `request.url` is the internal address,
8+
* so the forwarded headers win whenever they are present. Both can carry a
9+
* comma-separated chain when several proxies are involved; the first entry is
10+
* the one the client used.
11+
*/
12+
const firstHeaderValue = (value: string | null): string | undefined => {
13+
return value?.split(",")[0]?.trim() || undefined;
14+
};
15+
16+
export const getSiteOrigin = (request: Request): string => {
17+
const url = new URL(request.url);
18+
const host =
19+
firstHeaderValue(request.headers.get("x-forwarded-host")) ??
20+
firstHeaderValue(request.headers.get("host")) ??
21+
url.host;
22+
const protocol = firstHeaderValue(request.headers.get("x-forwarded-proto")) ?? url.protocol.replace(":", "");
23+
24+
return `${protocol}://${host}`;
25+
};
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import { createRootRoute, createRoute, createRouter } from "@tanstack/react-router";
2+
3+
import { buildSitemapXml, collectSitemapEntries } from "./sitemap";
4+
5+
/**
6+
* Builds a real router from a real route tree - the same shape the app ships -
7+
* so the collector is exercised against the router's own path table rather than
8+
* a hand-written stand-in that could drift from it.
9+
*/
10+
const createTestRouter = () => {
11+
const rootRoute = createRootRoute();
12+
13+
const indexRoute = createRoute({ getParentRoute: () => rootRoute, path: "/" });
14+
const pricingRoute = createRoute({
15+
getParentRoute: () => rootRoute,
16+
path: "/pricing",
17+
staticData: { sitemap: { changeFrequency: "weekly", priority: 0.8 } },
18+
});
19+
const settingsRoute = createRoute({
20+
getParentRoute: () => rootRoute,
21+
path: "/settings",
22+
staticData: { sitemap: false },
23+
});
24+
const postsRoute = createRoute({ getParentRoute: () => rootRoute, path: "/posts" });
25+
const postRoute = createRoute({ getParentRoute: () => postsRoute, path: "/$postId" });
26+
const feedRoute = createRoute({
27+
getParentRoute: () => rootRoute,
28+
path: "/feed.xml",
29+
server: { handlers: { GET: () => new Response("") } },
30+
});
31+
32+
return createRouter({
33+
routeTree: rootRoute.addChildren([
34+
indexRoute,
35+
pricingRoute,
36+
settingsRoute,
37+
postsRoute.addChildren([postRoute]),
38+
feedRoute,
39+
]),
40+
});
41+
};
42+
43+
describe("collectSitemapEntries", () => {
44+
it("lists every static page in the route tree", () => {
45+
// Arrange
46+
const router = createTestRouter();
47+
48+
// Act
49+
const entries = collectSitemapEntries(router.routesByPath);
50+
51+
// Assert
52+
expect(entries.map((entry) => entry.path)).toStrictEqual(["/", "/pricing"]);
53+
});
54+
55+
it("keeps routes opted out with `staticData.sitemap: false` out", () => {
56+
// Arrange
57+
const router = createTestRouter();
58+
59+
// Act
60+
const entries = collectSitemapEntries(router.routesByPath);
61+
62+
// Assert
63+
expect(entries.map((entry) => entry.path)).not.toContain("/settings");
64+
});
65+
66+
it("skips dynamic routes and the layouts that wrap them", () => {
67+
// Arrange
68+
const router = createTestRouter();
69+
70+
// Act
71+
const paths = collectSitemapEntries(router.routesByPath).map((entry) => entry.path);
72+
73+
// Assert
74+
expect(paths).not.toContain("/posts/$postId");
75+
expect(paths).not.toContain("/posts");
76+
});
77+
78+
it("skips routes that only answer with a server handler", () => {
79+
// Arrange
80+
const router = createTestRouter();
81+
82+
// Act
83+
const paths = collectSitemapEntries(router.routesByPath).map((entry) => entry.path);
84+
85+
// Assert
86+
expect(paths).not.toContain("/feed.xml");
87+
});
88+
89+
it("carries the route's own sitemap options over", () => {
90+
// Arrange
91+
const router = createTestRouter();
92+
93+
// Act
94+
const entries = collectSitemapEntries(router.routesByPath);
95+
96+
// Assert
97+
expect(entries).toContainEqual({ path: "/pricing", changeFrequency: "weekly", priority: 0.8 });
98+
});
99+
});
100+
101+
describe("buildSitemapXml", () => {
102+
it("renders absolute URLs under the given origin", () => {
103+
// Act
104+
const xml = buildSitemapXml("https://example.com", [{ path: "/" }, { path: "/pricing" }]);
105+
106+
// Assert
107+
expect(xml).toContain("<loc>https://example.com/</loc>");
108+
expect(xml).toContain("<loc>https://example.com/pricing</loc>");
109+
});
110+
111+
it("renders the optional per-entry hints", () => {
112+
// Act
113+
const xml = buildSitemapXml("https://example.com", [
114+
{
115+
path: "/pricing",
116+
changeFrequency: "weekly",
117+
priority: 0.8,
118+
lastModified: new Date("2026-01-01T00:00:00.000Z"),
119+
},
120+
]);
121+
122+
// Assert
123+
expect(xml).toContain("<lastmod>2026-01-01T00:00:00.000Z</lastmod>");
124+
expect(xml).toContain("<changefreq>weekly</changefreq>");
125+
expect(xml).toContain("<priority>0.8</priority>");
126+
});
127+
128+
it("escapes characters that would break the document", () => {
129+
// Act
130+
const xml = buildSitemapXml("https://example.com", [{ path: "/search?q=a&b" }]);
131+
132+
// Assert
133+
expect(xml).toContain("<loc>https://example.com/search?q=a&amp;b</loc>");
134+
});
135+
});
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import type { AnyRoute } from "@tanstack/router-core";
2+
3+
export type SitemapChangeFrequency = "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never";
4+
5+
export interface SitemapRouteOptions {
6+
changeFrequency?: SitemapChangeFrequency;
7+
/** 0.0 - 1.0, relative to the other pages of this site. Crawlers treat it as a hint. */
8+
priority?: number;
9+
lastModified?: Date | string;
10+
}
11+
12+
declare module "@tanstack/router-core" {
13+
interface StaticDataRouteOption {
14+
/**
15+
* How the route shows up in `/sitemap.xml`. Every static page is listed by
16+
* default, so this is only needed to tune an entry - or to set `false` and
17+
* keep a page out entirely, which is what anything behind a login wants:
18+
*
19+
* ```ts
20+
* export const Route = createFileRoute("/(private)/settings/")({
21+
* staticData: { sitemap: false },
22+
* });
23+
* ```
24+
*/
25+
sitemap?: SitemapRouteOptions | false;
26+
}
27+
}
28+
29+
export interface SitemapEntry extends SitemapRouteOptions {
30+
path: string;
31+
}
32+
33+
/**
34+
* `$postId`, `$`, `{-$locale}` - a crawler needs concrete URLs, and the values
35+
* that fill these live in a database this module cannot read. Routes that should
36+
* expand into real URLs have to be appended by whoever owns that data.
37+
*/
38+
const DYNAMIC_SEGMENT = /[$*{]/;
39+
40+
/**
41+
* Builds the sitemap straight from the router's own path table, so a new page
42+
* file is a new sitemap entry with nothing else to remember. Pass
43+
* `router.routesByPath` - it is keyed by full path and already has groups
44+
* (`(private)`) and pathless layouts (`_auth`) resolved away.
45+
*/
46+
export const collectSitemapEntries = (routesByPath: object): SitemapEntry[] => {
47+
// `routesByPath` is typed per route tree - an interface with one key per page
48+
// and no index signature - so it is read back as the record it is at runtime.
49+
const routes = Object.entries(routesByPath) as Array<[string, AnyRoute]>;
50+
const entries: SitemapEntry[] = [];
51+
52+
for (const [path, route] of routes) {
53+
const options = route.options.staticData?.sitemap;
54+
55+
// Opted out by hand - typically a page behind authentication.
56+
if (options === false) {
57+
continue;
58+
}
59+
60+
if (DYNAMIC_SEGMENT.test(path)) {
61+
continue;
62+
}
63+
64+
// A route with children is a layout: it wraps pages rather than being one,
65+
// and its own index child is already in the table under the same path.
66+
if (route.children?.length) {
67+
continue;
68+
}
69+
70+
// Server-only routes - `/sitemap.xml` and `/robots.txt` themselves - answer
71+
// with a handler instead of rendering a page, so they are not URLs to crawl.
72+
if (!route.options.component && route.options.server?.handlers) {
73+
continue;
74+
}
75+
76+
entries.push({ ...options, path: path || "/" });
77+
}
78+
79+
return entries.toSorted((a, b) => a.path.localeCompare(b.path));
80+
};
81+
82+
const XML_ESCAPES: Record<string, string> = {
83+
"&": "&amp;",
84+
"<": "&lt;",
85+
">": "&gt;",
86+
'"': "&quot;",
87+
"'": "&apos;",
88+
};
89+
90+
const escapeXml = (value: string): string => value.replace(/[&<>"']/g, (char) => XML_ESCAPES[char] ?? char);
91+
92+
const toIsoDate = (value: Date | string): string => (value instanceof Date ? value.toISOString() : value);
93+
94+
export const buildSitemapXml = (origin: string, entries: SitemapEntry[]): string => {
95+
const urls = entries.map(({ path, lastModified, changeFrequency, priority }) => {
96+
const lines = [` <loc>${escapeXml(`${origin}${path}`)}</loc>`];
97+
98+
if (lastModified) {
99+
lines.push(` <lastmod>${escapeXml(toIsoDate(lastModified))}</lastmod>`);
100+
}
101+
if (changeFrequency) {
102+
lines.push(` <changefreq>${changeFrequency}</changefreq>`);
103+
}
104+
if (priority !== undefined) {
105+
lines.push(` <priority>${priority}</priority>`);
106+
}
107+
108+
return ` <url>\n${lines.join("\n")}\n </url>`;
109+
});
110+
111+
return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`;
112+
};

0 commit comments

Comments
 (0)