diff --git a/CHANGELOG.md b/CHANGELOG.md index 65ef889d..5efc3552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ ### Changed -- Keyless recovery messages now link to `https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys` instead of `/app/api-keys`, so accounts created from them can be attributed to the MCP keyless free tier. Signed-in users still land on the API keys page. +- Keyless recovery messages now link to the caller's own signup link, `https://firecrawl.dev/k/` (a 12-character encrypted token), instead of `/app/api-keys`. The API issues the link (the `signup_url` of a keyless 429, or `signupUrl` from the eligibility check, which the server now also asks for when a keyless session calls an account-only tool) and the site decrypts it to MCP keyless attribution. When the API can't give one it sends the regular keyless signup link, which is relayed as is; with no API link at all the message uses the regular MCP signup link (`signin?utm_source=keyless&utm_medium=mcp`). Recovery payloads carry the link as `signup_url`. Signed-in users who open it land on the API keys page. - The search surface (`/v2/mcp-search`) now exposes `firecrawl_find_tools` and `firecrawl_scrape` alongside its six search tools, so agents can execute the Alexandria providers that `firecrawl_search` already returns. Both carry surface-scoped descriptions that name only tools registered on that surface, and Alexandria results there omit the `firecrawl_feedback` pointer. See docs/search-profile.md. ### Fixed diff --git a/gemini-extension.json b/gemini-extension.json index 795ae55a..a7c3c1d2 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "firecrawl", - "version": "3.26.0", + "version": "3.27.0", "description": "Official Firecrawl MCP for web search, scraping, crawling, and structured data extraction.", "mcpServers": { "firecrawl": { diff --git a/package.json b/package.json index 706cadeb..ad6f3050 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-mcp", - "version": "3.26.0", + "version": "3.27.0", "description": "MCP server for Firecrawl — search, scrape, and interact with the web, and search scientific papers. Supports both cloud and self-hosted instances. Features include web search, scraping, page interaction, batch processing, LLM-powered content analysis, and research paper search over biomedical and arXiv literature (PubMed, bioRxiv, medRxiv, arXiv) with citation-graph expansion and full-text reading.", "type": "module", "mcpName": "io.github.firecrawl/firecrawl-mcp-server", diff --git a/src/index.ts b/src/index.ts index 5c692454..e8dd29f6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -59,6 +59,7 @@ import { import { alexandriaOutput } from './alexandria-output'; import { registerDeveloperTools } from './developer'; import { extractSingleTrustedClientIp } from './keyless-client-ip'; +import { checkKeylessSignupUrl } from './keyless-signup-link'; import { registerMonitorTools } from './monitor'; import { registerResearchTools } from './research'; import { registerUsageTools } from './usage'; @@ -1458,11 +1459,48 @@ function isLocalKeylessStartup(): boolean { // FastMCP copies UserError.message onto both content[0].text and // structuredContent.message. Hosts forward the text block, not // structured next_actions, so bearer and OAuth recovery strings live here. -const KEYLESS_ACCOUNT_FIX = - 'Fix: Create an API key at https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys and then:\n- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp\nThen start a new session.'; -const KEYLESS_QUOTA_MESSAGE = `You've hit Firecrawl's free MCP rate limit. To continue using without limits, create a Firecrawl API key.\n\n${KEYLESS_ACCOUNT_FIX}`; -const KEYLESS_TOOL_MESSAGE = `This tool needs a Firecrawl account.\n\n${KEYLESS_ACCOUNT_FIX}`; -const KEYLESS_ACCESS_MESSAGE = `Anonymous keyless access is unavailable for this request.\n\n${KEYLESS_ACCOUNT_FIX}`; +// +// The signup link is the caller's own firecrawl.dev/k/ link, issued by +// the API (the 429 body's signup_url, or signupUrl from the eligibility check): +// a 12-character encrypted token the site decrypts to keyless attribution. +// When the API has no token to give it sends the regular keyless signin link, +// which is relayed as is; without any API link, the regular MCP signin link is used. +const KEYLESS_SIGNUP_FALLBACK_URL = + 'https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys'; +// Firecrawl-hosted links that fail the check are logged once each, so a change +// in the API's link format shows up instead of silently falling back. +const droppedKeylessSignupUrls = new Set(); + +/** An API-issued keyless signup link, or undefined for anything else. */ +function keylessSignupUrlFrom(value: unknown): string | undefined { + const check = checkKeylessSignupUrl(value); + if (check.ok) return check.url; + if ( + check.firecrawlHost && + typeof value === 'string' && + droppedKeylessSignupUrls.size < 50 && + !droppedKeylessSignupUrls.has(value) + ) { + droppedKeylessSignupUrls.add(value); + console.warn( + '[WARN]', + new Date().toISOString(), + 'Ignoring an unrecognized Firecrawl keyless signup link from the API; using the fallback', + { signupUrl: value } + ); + } + return undefined; +} + +function keylessAccountFix(signupUrl: string): string { + return `Fix: Create an API key at ${signupUrl} and then:\n- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp\nThen start a new session.`; +} +const keylessQuotaMessage = (signupUrl: string) => + `You've hit Firecrawl's free MCP rate limit. To continue using without limits, create a Firecrawl API key.\n\n${keylessAccountFix(signupUrl)}`; +const keylessToolMessage = (signupUrl: string) => + `This tool needs a Firecrawl account.\n\n${keylessAccountFix(signupUrl)}`; +const keylessAccessMessage = (signupUrl: string) => + `Anonymous keyless access is unavailable for this request.\n\n${keylessAccountFix(signupUrl)}`; const INVALID_API_KEY_MESSAGE = 'The Firecrawl API key is invalid or revoked.\nFix: Replace the key on the existing Firecrawl MCP server, then start a new session. Get an API key at https://www.firecrawl.dev/app/api-keys'; const INVALID_OAUTH_MESSAGE = @@ -1559,9 +1597,10 @@ async function runWithCredentialRecovery( function recoveryPayload( code: string, requestId: string = randomUUID(), - options: { retryAfterSeconds?: number } = {} + options: { retryAfterSeconds?: number; signupUrl?: string } = {} ): Record { const retryAfterSeconds = options.retryAfterSeconds; + const signupUrl = options.signupUrl ?? KEYLESS_SIGNUP_FALLBACK_URL; const isQuotaExhausted = code === 'KEYLESS_QUOTA_EXHAUSTED' || code === 'KEYLESS_LIMIT_REACHED'; const isToolUnavailable = code === 'KEYLESS_TOOL_NOT_AVAILABLE'; @@ -1578,11 +1617,11 @@ function recoveryPayload( code === 'CREDENTIAL_INVALID' ? INVALID_API_KEY_MESSAGE : isQuotaExhausted - ? KEYLESS_QUOTA_MESSAGE + ? keylessQuotaMessage(signupUrl) : isToolUnavailable - ? KEYLESS_TOOL_MESSAGE + ? keylessToolMessage(signupUrl) : isKeylessAccessUnavailable - ? KEYLESS_ACCESS_MESSAGE + ? keylessAccessMessage(signupUrl) : isKeylessEligibilityUnavailable ? 'The anonymous keyless eligibility check is temporarily unavailable. Retry shortly.' : 'This tool requires a Firecrawl account or API key.', @@ -1596,6 +1635,7 @@ function recoveryPayload( ...(isKeylessConversion || code === 'CREDENTIAL_INVALID' ? {} : { available_tools: [...KEYLESS_TOOL_NAMES] }), + ...(isKeylessConversion ? { signup_url: signupUrl } : {}), docs_url: MCP_CONNECTION_GUIDE_URL, ...(retryAfterSeconds ? { retry_after_seconds: retryAfterSeconds } : {}), ...(isKeylessEligibilityUnavailable @@ -1745,7 +1785,12 @@ function guardHostedTool( : undefined; if (code) { const requestId = randomUUID(); - const payload = recoveryPayload(code, requestId); + const payload = recoveryPayload(code, requestId, { + signupUrl: + code === 'KEYLESS_TOOL_NOT_AVAILABLE' + ? await hostedKeylessSignupUrl(session) + : undefined, + }); if (logActions) { emitActionLog(tool.name, 'error', session, new UserError(String(payload.message), payload), requestId, code); } @@ -1796,7 +1841,9 @@ function guardHostedTool( } if (isHostedKeylessSession(invocationSession) && !keylessTool) { const code = 'KEYLESS_TOOL_NOT_AVAILABLE'; - const payload = recoveryPayload(code, requestId); + const payload = recoveryPayload(code, requestId, { + signupUrl: await hostedKeylessSignupUrl(invocationSession), + }); if (logActions) emitActionLog(tool.name, 'error', invocationSession, new UserError(String(payload.message), payload), requestId, code); throw new UserError(String(payload.message), payload); } @@ -3011,6 +3058,7 @@ type KeylessEligibility = { reason?: string; retryAfterSeconds?: number; unavailable?: boolean; + signupUrl?: string; }; function keylessQuotaReason(reason: unknown): reason is 'requests' | 'credits' { @@ -3019,13 +3067,14 @@ function keylessQuotaReason(reason: unknown): reason is 'requests' | 'credits' { async function keylessEligible( clientIp: string, - origin: string + origin: string, + { signupLink = false }: { signupLink?: boolean } = {} ): Promise { const secret = process.env.KEYLESS_PROXY_SECRET; if (!secret) return { eligible: false, unavailable: true }; try { const response = await fetch( - `${resolveApiBaseUrl()}/v2/keyless/eligibility`, + `${resolveApiBaseUrl()}/v2/keyless/eligibility${signupLink ? '?signup_link=1' : ''}`, { headers: { ...originHeaders(origin), @@ -3045,11 +3094,31 @@ async function keylessEligible( ...(Number.isFinite(json?.retryAfterSeconds) && json.retryAfterSeconds > 0 ? { retryAfterSeconds: json.retryAfterSeconds } : {}), + ...(keylessSignupUrlFrom(json?.signupUrl) + ? { signupUrl: json.signupUrl } + : {}), }; } catch { return { eligible: false, unavailable: true }; } } + +/** + * The hosted keyless caller's own signup link, for recovery the API did not + * produce (a tool keyless sessions cannot use). Undefined falls back to the + * regular MCP signin link. + */ +async function hostedKeylessSignupUrl( + session?: SessionData +): Promise { + if (!session?.keylessClientIp) return undefined; + const eligibility = await keylessEligible( + session.keylessClientIp, + requestOrigin(undefined, session), + { signupLink: true } + ); + return eligibility.signupUrl; +} function isKeylessMode(session?: SessionData): boolean { if (hasCredential(session) || session?.credentialError) return false; if (process.env.CLOUD_SERVICE === 'true') { @@ -3084,6 +3153,7 @@ async function keylessPost( : 'KEYLESS_ACCESS_NOT_AVAILABLE'; const payload = recoveryPayload(code, session?.requestId, { retryAfterSeconds: eligibility.retryAfterSeconds, + signupUrl: eligibility.signupUrl, }); throw new UserError(String(payload.message), payload); } @@ -3118,6 +3188,7 @@ async function keylessPost( json.retry_after_seconds > 0 ? json.retry_after_seconds : undefined, + signupUrl: keylessSignupUrlFrom(json?.signup_url), }); const hints = readAgentHints(json); if (hints) payload.agent_hints = hints; diff --git a/src/keyless-signup-link.ts b/src/keyless-signup-link.ts new file mode 100644 index 00000000..6329bb7e --- /dev/null +++ b/src/keyless-signup-link.ts @@ -0,0 +1,56 @@ +// Keyless signup links the API may hand the MCP server to relay: the caller's +// own https://firecrawl.dev/k/ link, or the regular keyless signin link +// the API sends when it has no token. The URL is parsed and checked field by +// field, so parameter order and percent-encoding case don't matter, but only +// Firecrawl's own signup links are ever relayed. + +const SIGNUP_HOSTS = new Set(['firecrawl.dev', 'www.firecrawl.dev']); +const TOKEN_PATH = /^\/k\/[0-9abcdefghjkmnpqrstvwxyz]{12}$/; +const SURFACES = new Set(['api', 'mcp', 'cli']); +const SIGNIN_PARAMS = new Set(['utm_source', 'utm_medium', 'redirect']); +const SIGNIN_REDIRECT = '/app/api-keys'; + +/** Why a Firecrawl-hosted link was not relayed, for drift logging. */ +export type KeylessSignupUrlCheck = + | { ok: true; url: string } + | { ok: false; firecrawlHost: boolean }; + +export function checkKeylessSignupUrl(value: unknown): KeylessSignupUrlCheck { + if (typeof value !== 'string') return { ok: false, firecrawlHost: false }; + let url: URL; + try { + url = new URL(value); + } catch { + return { ok: false, firecrawlHost: false }; + } + const firecrawlHost = SIGNUP_HOSTS.has(url.hostname); + if ( + url.protocol !== 'https:' || + !firecrawlHost || + url.port || + url.username || + url.password || + url.hash + ) { + return { ok: false, firecrawlHost }; + } + if (TOKEN_PATH.test(url.pathname) && !url.search) { + return { ok: true, url: value }; + } + if (url.pathname === '/signin') { + const params = url.searchParams; + const keys = [...params.keys()]; + const known = + keys.every((key) => SIGNIN_PARAMS.has(key)) && + new Set(keys).size === keys.length; + if ( + known && + params.get('utm_source') === 'keyless' && + SURFACES.has(params.get('utm_medium') ?? '') && + (!params.has('redirect') || params.get('redirect') === SIGNIN_REDIRECT) + ) { + return { ok: true, url: value }; + } + } + return { ok: false, firecrawlHost }; +} diff --git a/tests/keyless-signup-link.test.mjs b/tests/keyless-signup-link.test.mjs new file mode 100644 index 00000000..d98b1b0e --- /dev/null +++ b/tests/keyless-signup-link.test.mjs @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { checkKeylessSignupUrl } from '../dist/keyless-signup-link.js'; + +const relayed = (value) => checkKeylessSignupUrl(value).ok; + +test('relays the caller\'s own /k token link on either Firecrawl host', () => { + assert.equal(relayed('https://firecrawl.dev/k/hrxch5c20tcs'), true); + assert.equal(relayed('https://www.firecrawl.dev/k/hrxch5c20tcs'), true); +}); + +test('relays the regular keyless signin link regardless of parameter order or encoding case', () => { + for (const url of [ + 'https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp', + 'https://firecrawl.dev/signin?utm_source=keyless&utm_medium=api', + 'https://www.firecrawl.dev/signin?utm_medium=cli&utm_source=keyless', + 'https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys', + 'https://www.firecrawl.dev/signin?redirect=%2fapp%2fapi-keys&utm_medium=mcp&utm_source=keyless', + ]) { + assert.equal(relayed(url), true, url); + } +}); + +test('rejects anything that is not one of Firecrawl\'s own signup links', () => { + for (const url of [ + 'https://evil.example/k/hrxch5c20tcs', + 'https://firecrawl.dev.evil.example/k/hrxch5c20tcs', + 'http://firecrawl.dev/k/hrxch5c20tcs', + 'https://firecrawl.dev:8443/k/hrxch5c20tcs', + 'https://user@firecrawl.dev/k/hrxch5c20tcs', + 'https://firecrawl.dev/k/hrxch5c20tcs?x=1', + 'https://firecrawl.dev/k/hrxch5c20tcs#frag', + 'https://firecrawl.dev/k/short', + 'https://firecrawl.dev/k/HRXCH5C20TCS', + 'https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=web', + 'https://www.firecrawl.dev/signin?utm_source=ads&utm_medium=mcp', + 'https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=https%3A%2F%2Fevil.example', + 'https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&next=%2Fx', + 'https://www.firecrawl.dev/signin?utm_source=keyless&utm_source=keyless&utm_medium=mcp', + 'https://www.firecrawl.dev/pricing?utm_source=keyless&utm_medium=mcp', + 'not a url', + 42, + ]) { + assert.equal(relayed(url), false, String(url)); + } +}); + +test('flags rejected Firecrawl-hosted links so format drift can be logged', () => { + assert.deepEqual( + checkKeylessSignupUrl('https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=web'), + { ok: false, firecrawlHost: true } + ); + assert.deepEqual(checkKeylessSignupUrl('https://evil.example/k/hrxch5c20tcs'), { + ok: false, + firecrawlHost: false, + }); +}); diff --git a/tests/mcp-alexandria-auth.test.mjs b/tests/mcp-alexandria-auth.test.mjs index bd14cc11..21a8b85c 100644 --- a/tests/mcp-alexandria-auth.test.mjs +++ b/tests/mcp-alexandria-auth.test.mjs @@ -115,7 +115,8 @@ test('hosted keyless sessions never reach the Exchange; an API key header does', } assert.equal( backend.requests.some( - (request) => request.url !== '/v2/keyless/eligibility' + // The account-only recovery asks eligibility for the caller's signup link. + (request) => request.url.split('?')[0] !== '/v2/keyless/eligibility' ), false, 'keyless sessions must not reach /v2/scrape, /v2/search, or /exchange/*' diff --git a/tests/mcp-smoke.test.mjs b/tests/mcp-smoke.test.mjs index b6415f60..c674c5c5 100644 --- a/tests/mcp-smoke.test.mjs +++ b/tests/mcp-smoke.test.mjs @@ -70,11 +70,21 @@ function assertServerGeneratedRequestId(payload, untrustedValues = []) { return payload.request_id; } -const KEYLESS_ACCOUNT_FIX = - 'Fix: Create an API key at https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys and then:\n- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp\nThen start a new session.'; -const KEYLESS_QUOTA_MESSAGE = `You've hit Firecrawl's free MCP rate limit. To continue using without limits, create a Firecrawl API key.\n\n${KEYLESS_ACCOUNT_FIX}`; -const KEYLESS_TOOL_MESSAGE = `This tool needs a Firecrawl account.\n\n${KEYLESS_ACCOUNT_FIX}`; -const KEYLESS_ACCESS_MESSAGE = `Anonymous keyless access is unavailable for this request.\n\n${KEYLESS_ACCOUNT_FIX}`; +// Without an API-issued link, recovery falls back to the regular MCP signin link. +const KEYLESS_SIGNUP_FALLBACK_URL = + 'https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys'; +const API_REGULAR_SIGNUP_URL = + 'https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp'; +const OWN_SIGNUP_URL = 'https://firecrawl.dev/k/hrxch5c20tcs'; +const keylessAccountFix = (signupUrl = KEYLESS_SIGNUP_FALLBACK_URL) => + `Fix: Create an API key at ${signupUrl} and then:\n- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp\nThen start a new session.`; +const keylessQuotaMessage = (signupUrl) => + `You've hit Firecrawl's free MCP rate limit. To continue using without limits, create a Firecrawl API key.\n\n${keylessAccountFix(signupUrl)}`; +const keylessToolMessage = (signupUrl) => + `This tool needs a Firecrawl account.\n\n${keylessAccountFix(signupUrl)}`; +const KEYLESS_QUOTA_MESSAGE = keylessQuotaMessage(); +const KEYLESS_TOOL_MESSAGE = keylessToolMessage(); +const KEYLESS_ACCESS_MESSAGE = `Anonymous keyless access is unavailable for this request.\n\n${keylessAccountFix()}`; const INVALID_API_KEY_MESSAGE = 'The Firecrawl API key is invalid or revoked.\nFix: Replace the key on the existing Firecrawl MCP server, then start a new session. Get an API key at https://www.firecrawl.dev/app/api-keys'; const INVALID_OAUTH_MESSAGE = @@ -657,8 +667,8 @@ async function startFakeFirecrawlBackend(options = {}) { } // Keyless free-tier eligibility (secret-gated, read-only). - if (req.method === 'GET' && req.url === '/v2/keyless/eligibility') { - const response = keylessEligibilityResponse ?? { + if (req.method === 'GET' && req.url.split('?')[0] === '/v2/keyless/eligibility') { + const response = keylessEligibilityResponse?.(req.url) ?? { status: 200, body: { eligible: keylessEligible }, }; @@ -2282,7 +2292,7 @@ test('HTTP cloud transport serves an eligible keyless client and forwards its IP test('HTTP cloud keyless returns retry recovery when eligibility is unavailable', async (t) => { const backend = await startFakeFirecrawlBackend({ - keylessEligibilityResponse: { status: 503, body: { error: 'unavailable' } }, + keylessEligibilityResponse: () => ({ status: 503, body: { error: 'unavailable' } }), }); t.after(() => backend.close()); const port = await getFreePort(); @@ -2385,10 +2395,10 @@ test('HTTP cloud keyless quota stays 200 isError without inlining retry_after_se [ 'eligibility-exhausted', { - keylessEligibilityResponse: { + keylessEligibilityResponse: () => ({ status: 200, body: { eligible: false, reason: 'credits', retryAfterSeconds: 42 }, - }, + }), }, 'KEYLESS_QUOTA_EXHAUSTED', 42, @@ -2434,6 +2444,299 @@ test('HTTP cloud keyless quota stays 200 isError without inlining retry_after_se } }); +test('HTTP cloud keyless recovery relays the caller\'s own /k link from the API', async (t) => { + for (const [label, backendOptions] of [ + [ + 'eligibility-exhausted', + { + keylessEligibilityResponse: () => ({ + status: 200, + body: { eligible: false, reason: 'requests', signupUrl: OWN_SIGNUP_URL }, + }), + }, + ], + [ + 'core-429', + { + keylessEligible: true, + searchResponse: { + status: 429, + body: { error: 'limit', reason: 'credits', signup_url: OWN_SIGNUP_URL }, + }, + }, + ], + ]) { + const backend = await startFakeFirecrawlBackend(backendOptions); + const port = await getFreePort(); + const child = spawnServer({ + CLOUD_SERVICE: 'true', + FASTMCP_ENDPOINT: '/v2/mcp', + FIRECRAWL_API_URL: backend.url, + HTTP_STREAMABLE_SERVER: 'true', + KEYLESS_PROXY_SECRET: 'keyless-secret', + PORT: String(port), + }); + let cleanedUp = false; + const cleanup = async () => { + if (cleanedUp) return; + cleanedUp = true; + await stopChild(child); + await backend.close(); + }; + t.after(cleanup); + await waitForHealth(port, child); + const response = await httpToolCall(port, { + id: `keyless-own-link-${label}`, + headers: { 'x-forwarded-for': '8.8.8.7' }, + params: { arguments: { limit: 1, query: 'example domain' }, name: 'firecrawl_search' }, + }); + const result = parseSseJson(await response.text()).result; + assertKeylessAccountRecovery(result, { + code: 'KEYLESS_QUOTA_EXHAUSTED', + message: keylessQuotaMessage(OWN_SIGNUP_URL), + }); + assert.equal(result.structuredContent.signup_url, OWN_SIGNUP_URL, label); + assert.doesNotMatch(result.content[0].text, /utm_/, label); + await cleanup(); + } +}); + +test('HTTP cloud keyless recovery relays the API\'s regular signup link when it has no /k link', async (t) => { + for (const [label, backendOptions] of [ + [ + 'eligibility-exhausted', + { + keylessEligibilityResponse: () => ({ + status: 200, + body: { eligible: false, reason: 'requests', signupUrl: API_REGULAR_SIGNUP_URL }, + }), + }, + ], + [ + 'core-429', + { + keylessEligible: true, + searchResponse: { + status: 429, + body: { error: 'limit', reason: 'credits', signup_url: API_REGULAR_SIGNUP_URL }, + }, + }, + ], + ]) { + const backend = await startFakeFirecrawlBackend(backendOptions); + const port = await getFreePort(); + const child = spawnServer({ + CLOUD_SERVICE: 'true', + FASTMCP_ENDPOINT: '/v2/mcp', + FIRECRAWL_API_URL: backend.url, + HTTP_STREAMABLE_SERVER: 'true', + KEYLESS_PROXY_SECRET: 'keyless-secret', + PORT: String(port), + }); + let cleanedUp = false; + const cleanup = async () => { + if (cleanedUp) return; + cleanedUp = true; + await stopChild(child); + await backend.close(); + }; + t.after(cleanup); + await waitForHealth(port, child); + const response = await httpToolCall(port, { + id: `keyless-regular-link-${label}`, + headers: { 'x-forwarded-for': '8.8.8.7' }, + params: { arguments: { limit: 1, query: 'example domain' }, name: 'firecrawl_search' }, + }); + const result = parseSseJson(await response.text()).result; + assertKeylessAccountRecovery(result, { + code: 'KEYLESS_QUOTA_EXHAUSTED', + message: keylessQuotaMessage(API_REGULAR_SIGNUP_URL), + }); + assert.equal(result.structuredContent.signup_url, API_REGULAR_SIGNUP_URL, label); + await cleanup(); + } +}); + +test('HTTP cloud keyless recovery ignores a signup link that is not a firecrawl.dev/k link', async (t) => { + const untrustedUrl = 'https://evil.example/k/hrxch5c20tcs'; + for (const [label, backendOptions] of [ + [ + 'eligibility-exhausted', + { + keylessEligibilityResponse: () => ({ + status: 200, + body: { eligible: false, reason: 'requests', signupUrl: untrustedUrl }, + }), + }, + ], + [ + 'core-429', + { + keylessEligible: true, + searchResponse: { + status: 429, + body: { error: 'limit', reason: 'credits', signup_url: untrustedUrl }, + }, + }, + ], + ]) { + const backend = await startFakeFirecrawlBackend(backendOptions); + const port = await getFreePort(); + const child = spawnServer({ + CLOUD_SERVICE: 'true', + FASTMCP_ENDPOINT: '/v2/mcp', + FIRECRAWL_API_URL: backend.url, + HTTP_STREAMABLE_SERVER: 'true', + KEYLESS_PROXY_SECRET: 'keyless-secret', + PORT: String(port), + }); + let cleanedUp = false; + const cleanup = async () => { + if (cleanedUp) return; + cleanedUp = true; + await stopChild(child); + await backend.close(); + }; + t.after(cleanup); + await waitForHealth(port, child); + const response = await httpToolCall(port, { + id: `keyless-untrusted-link-${label}`, + headers: { 'x-forwarded-for': '8.8.8.7' }, + params: { arguments: { limit: 1, query: 'example domain' }, name: 'firecrawl_search' }, + }); + const result = parseSseJson(await response.text()).result; + assertKeylessAccountRecovery(result, { + code: 'KEYLESS_QUOTA_EXHAUSTED', + message: KEYLESS_QUOTA_MESSAGE, + }); + assert.equal(result.structuredContent.signup_url, KEYLESS_SIGNUP_FALLBACK_URL, label); + assert.doesNotMatch(result.content[0].text, /evil\.example/, label); + await cleanup(); + } +}); + +test('HTTP cloud keyless account-only tool asks the API for the caller\'s own link', async (t) => { + const backend = await startFakeFirecrawlBackend({ + keylessEligibilityResponse: (url) => ({ + status: 200, + body: url.includes('signup_link=1') + ? { eligible: true, signupUrl: OWN_SIGNUP_URL } + : { eligible: true }, + }), + }); + t.after(() => backend.close()); + const port = await getFreePort(); + const child = spawnServer({ + CLOUD_SERVICE: 'true', + FASTMCP_ENDPOINT: '/v2/mcp', + FIRECRAWL_API_URL: backend.url, + HTTP_STREAMABLE_SERVER: 'true', + KEYLESS_PROXY_SECRET: 'keyless-secret', + PORT: String(port), + }); + t.after(() => stopChild(child)); + await waitForHealth(port, child); + + const response = await httpToolCall(port, { + id: 'keyless-account-only-own-link', + headers: { 'x-forwarded-for': '8.8.8.7' }, + params: { arguments: { url: 'https://example.com/' }, name: 'firecrawl_crawl' }, + }); + const result = parseSseJson(await response.text()).result; + assertKeylessAccountRecovery(result, { + code: 'KEYLESS_TOOL_NOT_AVAILABLE', + message: keylessToolMessage(OWN_SIGNUP_URL), + }); + const linkRequest = backend.requests.find((r) => + r.url === '/v2/keyless/eligibility?signup_link=1' + ); + assert.ok(linkRequest); + assert.equal(linkRequest.headers['x-firecrawl-keyless-ip'], '8.8.8.7'); + assert.equal(linkRequest.headers['x-firecrawl-keyless-secret'], 'keyless-secret'); + assert.equal(backend.requests.some((r) => r.url === '/v2/crawl'), false); +}); + +test("HTTP cloud keyless account-only tool relays the API's regular link and drops an untrusted one", async (t) => { + for (const [label, signupUrl, expected] of [ + ['regular-link', API_REGULAR_SIGNUP_URL, API_REGULAR_SIGNUP_URL], + [ + 'regular-link-bare-host', + 'https://firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp', + 'https://firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp', + ], + [ + 'untrusted-host', + 'https://evil.example/k/hrxch5c20tcs', + KEYLESS_SIGNUP_FALLBACK_URL, + ], + [ + 'legacy-short-id', + 'https://firecrawl.dev/k/7fq2xab9', + KEYLESS_SIGNUP_FALLBACK_URL, + ], + ]) { + // The link comes only from the signup_link=1 check, so a relay here can + // only be the account-only path's. + const backend = await startFakeFirecrawlBackend({ + keylessEligibilityResponse: (url) => ({ + status: 200, + body: url.includes('signup_link=1') + ? { eligible: true, signupUrl } + : { eligible: true }, + }), + }); + const port = await getFreePort(); + const child = spawnServer({ + CLOUD_SERVICE: 'true', + FASTMCP_ENDPOINT: '/v2/mcp', + FIRECRAWL_API_URL: backend.url, + HTTP_STREAMABLE_SERVER: 'true', + KEYLESS_PROXY_SECRET: 'keyless-secret', + PORT: String(port), + }); + let cleanedUp = false; + const cleanup = async () => { + if (cleanedUp) return; + cleanedUp = true; + await stopChild(child); + await backend.close(); + }; + t.after(cleanup); + await waitForHealth(port, child); + const response = await httpToolCall(port, { + id: `keyless-account-only-${label}`, + headers: { 'x-forwarded-for': '8.8.8.7' }, + params: { + arguments: { url: 'https://example.com/' }, + name: 'firecrawl_crawl', + }, + }); + const result = parseSseJson(await response.text()).result; + assertKeylessAccountRecovery(result, { + code: 'KEYLESS_TOOL_NOT_AVAILABLE', + message: keylessToolMessage(expected), + }); + assert.equal(result.structuredContent.signup_url, expected, label); + assert.ok( + backend.requests.some( + (r) => r.url === '/v2/keyless/eligibility?signup_link=1' + ), + label + ); + assert.doesNotMatch( + result.content[0].text, + /evil\.example|7fq2xab9/, + label + ); + assert.equal( + backend.requests.some((r) => r.url === '/v2/crawl'), + false, + label + ); + await cleanup(); + } +}); + test('HTTP cloud keyless Parse completes both phases without credentials and forwards redactPII', async (t) => { const backend = await startFakeFirecrawlBackend({ keylessEligible: true }); t.after(() => backend.close()); diff --git a/tsup.config.ts b/tsup.config.ts index 55954a18..c6e3234e 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -7,6 +7,7 @@ export default defineConfig({ 'src/agent-hints.ts', 'src/origin.ts', 'src/introspection-cache.ts', + 'src/keyless-signup-link.ts', ], format: ['esm'], platform: 'node',