Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

### Changed

- Keyless recovery messages now link to `https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys` instead of `/app/api-keys`, so accounts created from them can be attributed to the MCP keyless free tier. Signed-in users still land on the API keys page.
- Keyless recovery messages now link to the caller's own short signup link, `https://firecrawl.dev/k/<id>`, instead of `/app/api-keys`. The API issues the link (the `signup_url` of a keyless 429, or `signupUrl` from the eligibility check, which the server now also asks for when a keyless session calls an account-only tool) and the site resolves it to MCP keyless attribution. Without one the message uses `https://firecrawl.dev/k`. Recovery payloads carry the link as `signup_url`. Signed-in users who open it land on the API keys page.
- The search surface (`/v2/mcp-search`) now exposes `firecrawl_find_tools` and `firecrawl_scrape` alongside its six search tools, so agents can execute the Alexandria providers that `firecrawl_search` already returns. Both carry surface-scoped descriptions that name only tools registered on that surface, and Alexandria results there omit the `firecrawl_feedback` pointer. See docs/search-profile.md.

### Fixed
Expand Down
78 changes: 65 additions & 13 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1458,11 +1458,31 @@ function isLocalKeylessStartup(): boolean {
// FastMCP copies UserError.message onto both content[0].text and
// structuredContent.message. Hosts forward the text block, not
// structured next_actions, so bearer and OAuth recovery strings live here.
const KEYLESS_ACCOUNT_FIX =
'Fix: Create an API key at https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys and then:\n- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp\nThen start a new session.';
const KEYLESS_QUOTA_MESSAGE = `You've hit Firecrawl's free MCP rate limit. To continue using without limits, create a Firecrawl API key.\n\n${KEYLESS_ACCOUNT_FIX}`;
const KEYLESS_TOOL_MESSAGE = `This tool needs a Firecrawl account.\n\n${KEYLESS_ACCOUNT_FIX}`;
const KEYLESS_ACCESS_MESSAGE = `Anonymous keyless access is unavailable for this request.\n\n${KEYLESS_ACCOUNT_FIX}`;
//
// The signup link is the caller's own opaque firecrawl.dev/k/<id> link, issued
// by the API (the 429 body's signup_url, or signupUrl from the eligibility
// check), which resolves to keyless/mcp attribution on the site. Without one,
// the bare /k link still tags the signup as keyless.
const KEYLESS_SIGNUP_FALLBACK_URL = 'https://firecrawl.dev/k';
const KEYLESS_SIGNUP_URL_PATTERN =
/^https:\/\/(?:www\.)?firecrawl\.dev\/k(?:\/[0-9abcdefghjkmnpqrstvwxyz]{8})?$/;

/** An API-issued keyless signup link, or undefined for anything else. */
function keylessSignupUrlFrom(value: unknown): string | undefined {
return typeof value === 'string' && KEYLESS_SIGNUP_URL_PATTERN.test(value)
? value
: undefined;
}

function keylessAccountFix(signupUrl: string): string {
return `Fix: Create an API key at ${signupUrl} and then:\n- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp\nThen start a new session.`;
}
const keylessQuotaMessage = (signupUrl: string) =>
`You've hit Firecrawl's free MCP rate limit. To continue using without limits, create a Firecrawl API key.\n\n${keylessAccountFix(signupUrl)}`;
const keylessToolMessage = (signupUrl: string) =>
`This tool needs a Firecrawl account.\n\n${keylessAccountFix(signupUrl)}`;
const keylessAccessMessage = (signupUrl: string) =>
`Anonymous keyless access is unavailable for this request.\n\n${keylessAccountFix(signupUrl)}`;
const INVALID_API_KEY_MESSAGE =
'The Firecrawl API key is invalid or revoked.\nFix: Replace the key on the existing Firecrawl MCP server, then start a new session. Get an API key at https://www.firecrawl.dev/app/api-keys';
const INVALID_OAUTH_MESSAGE =
Expand Down Expand Up @@ -1559,9 +1579,10 @@ async function runWithCredentialRecovery<T>(
function recoveryPayload(
code: string,
requestId: string = randomUUID(),
options: { retryAfterSeconds?: number } = {}
options: { retryAfterSeconds?: number; signupUrl?: string } = {}
): Record<string, unknown> {
const retryAfterSeconds = options.retryAfterSeconds;
const signupUrl = options.signupUrl ?? KEYLESS_SIGNUP_FALLBACK_URL;
const isQuotaExhausted =
code === 'KEYLESS_QUOTA_EXHAUSTED' || code === 'KEYLESS_LIMIT_REACHED';
const isToolUnavailable = code === 'KEYLESS_TOOL_NOT_AVAILABLE';
Expand All @@ -1578,11 +1599,11 @@ function recoveryPayload(
code === 'CREDENTIAL_INVALID'
? INVALID_API_KEY_MESSAGE
: isQuotaExhausted
? KEYLESS_QUOTA_MESSAGE
? keylessQuotaMessage(signupUrl)
: isToolUnavailable
? KEYLESS_TOOL_MESSAGE
? keylessToolMessage(signupUrl)
: isKeylessAccessUnavailable
? KEYLESS_ACCESS_MESSAGE
? keylessAccessMessage(signupUrl)
: isKeylessEligibilityUnavailable
? 'The anonymous keyless eligibility check is temporarily unavailable. Retry shortly.'
: 'This tool requires a Firecrawl account or API key.',
Expand All @@ -1596,6 +1617,7 @@ function recoveryPayload(
...(isKeylessConversion || code === 'CREDENTIAL_INVALID'
? {}
: { available_tools: [...KEYLESS_TOOL_NAMES] }),
...(isKeylessConversion ? { signup_url: signupUrl } : {}),
docs_url: MCP_CONNECTION_GUIDE_URL,
...(retryAfterSeconds ? { retry_after_seconds: retryAfterSeconds } : {}),
...(isKeylessEligibilityUnavailable
Expand Down Expand Up @@ -1745,7 +1767,12 @@ function guardHostedTool(
: undefined;
if (code) {
const requestId = randomUUID();
const payload = recoveryPayload(code, requestId);
const payload = recoveryPayload(code, requestId, {
signupUrl:
code === 'KEYLESS_TOOL_NOT_AVAILABLE'
? await hostedKeylessSignupUrl(session)
: undefined,
});
if (logActions) {
emitActionLog(tool.name, 'error', session, new UserError(String(payload.message), payload), requestId, code);
}
Expand Down Expand Up @@ -1796,7 +1823,9 @@ function guardHostedTool(
}
if (isHostedKeylessSession(invocationSession) && !keylessTool) {
const code = 'KEYLESS_TOOL_NOT_AVAILABLE';
const payload = recoveryPayload(code, requestId);
const payload = recoveryPayload(code, requestId, {
signupUrl: await hostedKeylessSignupUrl(invocationSession),
});
if (logActions) emitActionLog(tool.name, 'error', invocationSession, new UserError(String(payload.message), payload), requestId, code);
throw new UserError(String(payload.message), payload);
}
Expand Down Expand Up @@ -3011,6 +3040,7 @@ type KeylessEligibility = {
reason?: string;
retryAfterSeconds?: number;
unavailable?: boolean;
signupUrl?: string;
};

function keylessQuotaReason(reason: unknown): reason is 'requests' | 'credits' {
Expand All @@ -3019,13 +3049,14 @@ function keylessQuotaReason(reason: unknown): reason is 'requests' | 'credits' {

async function keylessEligible(
clientIp: string,
origin: string
origin: string,
{ signupLink = false }: { signupLink?: boolean } = {}
): Promise<KeylessEligibility> {
const secret = process.env.KEYLESS_PROXY_SECRET;
if (!secret) return { eligible: false, unavailable: true };
try {
const response = await fetch(
`${resolveApiBaseUrl()}/v2/keyless/eligibility`,
`${resolveApiBaseUrl()}/v2/keyless/eligibility${signupLink ? '?signup_link=1' : ''}`,
{
headers: {
...originHeaders(origin),
Expand All @@ -3045,11 +3076,30 @@ async function keylessEligible(
...(Number.isFinite(json?.retryAfterSeconds) && json.retryAfterSeconds > 0
? { retryAfterSeconds: json.retryAfterSeconds }
: {}),
...(keylessSignupUrlFrom(json?.signupUrl)
? { signupUrl: json.signupUrl }
: {}),
};
} catch {
return { eligible: false, unavailable: true };
}
}

/**
* The hosted keyless caller's own signup link, for recovery the API did not
* produce (a tool keyless sessions cannot use). Undefined falls back to /k.
*/
async function hostedKeylessSignupUrl(
session?: SessionData
): Promise<string | undefined> {
if (!session?.keylessClientIp) return undefined;
const eligibility = await keylessEligible(
session.keylessClientIp,
requestOrigin(undefined, session),
{ signupLink: true }
);
return eligibility.signupUrl;
}
function isKeylessMode(session?: SessionData): boolean {
if (hasCredential(session) || session?.credentialError) return false;
if (process.env.CLOUD_SERVICE === 'true') {
Expand Down Expand Up @@ -3084,6 +3134,7 @@ async function keylessPost(
: 'KEYLESS_ACCESS_NOT_AVAILABLE';
const payload = recoveryPayload(code, session?.requestId, {
retryAfterSeconds: eligibility.retryAfterSeconds,
signupUrl: eligibility.signupUrl,
});
throw new UserError(String(payload.message), payload);
}
Expand Down Expand Up @@ -3118,6 +3169,7 @@ async function keylessPost(
json.retry_after_seconds > 0
? json.retry_after_seconds
: undefined,
signupUrl: keylessSignupUrlFrom(json?.signup_url),
});
const hints = readAgentHints(json);
if (hints) payload.agent_hints = hints;
Expand Down
2 changes: 1 addition & 1 deletion tests/helpers/exchange-mcp.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { startFakeExchangeApi } from './exchange-api.mjs';
const EXCHANGE_KEY_REQUIRED_MESSAGE =
'Alexandria requires an API key on a team with Alexandria access';
const KEYLESS_TOOL_MESSAGE =
'This tool needs a Firecrawl account.\n\nFix: Create an API key at https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys and then:\n- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp\nThen start a new session.';
'This tool needs a Firecrawl account.\n\nFix: Create an API key at https://firecrawl.dev/k and then:\n- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp\nThen start a new session.';

async function getFreePort() {
const server = net.createServer();
Expand Down
3 changes: 2 additions & 1 deletion tests/mcp-alexandria-auth.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ test('hosted keyless sessions never reach the Exchange; an API key header does',
}
assert.equal(
backend.requests.some(
(request) => request.url !== '/v2/keyless/eligibility'
// The account-only recovery asks eligibility for the caller's signup link.
(request) => request.url.split('?')[0] !== '/v2/keyless/eligibility'
),
false,
'keyless sessions must not reach /v2/scrape, /v2/search, or /exchange/*'
Expand Down
163 changes: 153 additions & 10 deletions tests/mcp-smoke.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,18 @@ function assertServerGeneratedRequestId(payload, untrustedValues = []) {
return payload.request_id;
}

const KEYLESS_ACCOUNT_FIX =
'Fix: Create an API key at https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=mcp&redirect=%2Fapp%2Fapi-keys and then:\n- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp\nThen start a new session.';
const KEYLESS_QUOTA_MESSAGE = `You've hit Firecrawl's free MCP rate limit. To continue using without limits, create a Firecrawl API key.\n\n${KEYLESS_ACCOUNT_FIX}`;
const KEYLESS_TOOL_MESSAGE = `This tool needs a Firecrawl account.\n\n${KEYLESS_ACCOUNT_FIX}`;
const KEYLESS_ACCESS_MESSAGE = `Anonymous keyless access is unavailable for this request.\n\n${KEYLESS_ACCOUNT_FIX}`;
// Without an API-issued link, recovery falls back to the bare /k link.
const KEYLESS_SIGNUP_FALLBACK_URL = 'https://firecrawl.dev/k';
const OWN_SIGNUP_URL = 'https://firecrawl.dev/k/7fq2xab9';
const keylessAccountFix = (signupUrl = KEYLESS_SIGNUP_FALLBACK_URL) =>
`Fix: Create an API key at ${signupUrl} and then:\n- Set the header: Authorization: Bearer YOUR_API_KEY on https://mcp.firecrawl.dev/v2/mcp\nThen start a new session.`;
const keylessQuotaMessage = (signupUrl) =>
`You've hit Firecrawl's free MCP rate limit. To continue using without limits, create a Firecrawl API key.\n\n${keylessAccountFix(signupUrl)}`;
const keylessToolMessage = (signupUrl) =>
`This tool needs a Firecrawl account.\n\n${keylessAccountFix(signupUrl)}`;
const KEYLESS_QUOTA_MESSAGE = keylessQuotaMessage();
const KEYLESS_TOOL_MESSAGE = keylessToolMessage();
const KEYLESS_ACCESS_MESSAGE = `Anonymous keyless access is unavailable for this request.\n\n${keylessAccountFix()}`;
const INVALID_API_KEY_MESSAGE =
'The Firecrawl API key is invalid or revoked.\nFix: Replace the key on the existing Firecrawl MCP server, then start a new session. Get an API key at https://www.firecrawl.dev/app/api-keys';
const INVALID_OAUTH_MESSAGE =
Expand Down Expand Up @@ -657,8 +664,8 @@ async function startFakeFirecrawlBackend(options = {}) {
}

// Keyless free-tier eligibility (secret-gated, read-only).
if (req.method === 'GET' && req.url === '/v2/keyless/eligibility') {
const response = keylessEligibilityResponse ?? {
if (req.method === 'GET' && req.url.split('?')[0] === '/v2/keyless/eligibility') {
const response = keylessEligibilityResponse?.(req.url) ?? {
status: 200,
body: { eligible: keylessEligible },
};
Expand Down Expand Up @@ -2282,7 +2289,7 @@ test('HTTP cloud transport serves an eligible keyless client and forwards its IP

test('HTTP cloud keyless returns retry recovery when eligibility is unavailable', async (t) => {
const backend = await startFakeFirecrawlBackend({
keylessEligibilityResponse: { status: 503, body: { error: 'unavailable' } },
keylessEligibilityResponse: () => ({ status: 503, body: { error: 'unavailable' } }),
});
t.after(() => backend.close());
const port = await getFreePort();
Expand Down Expand Up @@ -2385,10 +2392,10 @@ test('HTTP cloud keyless quota stays 200 isError without inlining retry_after_se
[
'eligibility-exhausted',
{
keylessEligibilityResponse: {
keylessEligibilityResponse: () => ({
status: 200,
body: { eligible: false, reason: 'credits', retryAfterSeconds: 42 },
},
}),
},
'KEYLESS_QUOTA_EXHAUSTED',
42,
Expand Down Expand Up @@ -2434,6 +2441,142 @@ test('HTTP cloud keyless quota stays 200 isError without inlining retry_after_se
}
});

test('HTTP cloud keyless recovery relays the caller\'s own /k link from the API', async (t) => {
for (const [label, backendOptions] of [
[
'eligibility-exhausted',
{
keylessEligibilityResponse: () => ({
status: 200,
body: { eligible: false, reason: 'requests', signupUrl: OWN_SIGNUP_URL },
}),
},
],
[
'core-429',
{
keylessEligible: true,
searchResponse: {
status: 429,
body: { error: 'limit', reason: 'credits', signup_url: OWN_SIGNUP_URL },
},
},
],
]) {
const backend = await startFakeFirecrawlBackend(backendOptions);
const port = await getFreePort();
const child = spawnServer({
CLOUD_SERVICE: 'true',
FASTMCP_ENDPOINT: '/v2/mcp',
FIRECRAWL_API_URL: backend.url,
HTTP_STREAMABLE_SERVER: 'true',
KEYLESS_PROXY_SECRET: 'keyless-secret',
PORT: String(port),
});
let cleanedUp = false;
const cleanup = async () => {
if (cleanedUp) return;
cleanedUp = true;
await stopChild(child);
await backend.close();
};
t.after(cleanup);
await waitForHealth(port, child);
const response = await httpToolCall(port, {
id: `keyless-own-link-${label}`,
headers: { 'x-forwarded-for': '8.8.8.7' },
params: { arguments: { limit: 1, query: 'example domain' }, name: 'firecrawl_search' },
});
const result = parseSseJson(await response.text()).result;
assertKeylessAccountRecovery(result, {
code: 'KEYLESS_QUOTA_EXHAUSTED',
message: keylessQuotaMessage(OWN_SIGNUP_URL),
});
assert.equal(result.structuredContent.signup_url, OWN_SIGNUP_URL, label);
assert.doesNotMatch(result.content[0].text, /utm_/, label);
await cleanup();
}
});

test('HTTP cloud keyless recovery ignores a signup link that is not a firecrawl.dev/k link', async (t) => {
const backend = await startFakeFirecrawlBackend({
keylessEligible: true,
searchResponse: {
status: 429,
body: {
error: 'limit',
reason: 'credits',
signup_url: 'https://evil.example/k/7fq2xab9',
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Outdated
},
},
});
t.after(() => backend.close());
const port = await getFreePort();
const child = spawnServer({
CLOUD_SERVICE: 'true',
FASTMCP_ENDPOINT: '/v2/mcp',
FIRECRAWL_API_URL: backend.url,
HTTP_STREAMABLE_SERVER: 'true',
KEYLESS_PROXY_SECRET: 'keyless-secret',
PORT: String(port),
});
t.after(() => stopChild(child));
await waitForHealth(port, child);

const response = await httpToolCall(port, {
id: 'keyless-untrusted-link',
headers: { 'x-forwarded-for': '8.8.8.7' },
params: { arguments: { limit: 1, query: 'example domain' }, name: 'firecrawl_search' },
});
const result = parseSseJson(await response.text()).result;
assertKeylessAccountRecovery(result, {
code: 'KEYLESS_QUOTA_EXHAUSTED',
message: KEYLESS_QUOTA_MESSAGE,
});
assert.equal(result.structuredContent.signup_url, KEYLESS_SIGNUP_FALLBACK_URL);
});

test('HTTP cloud keyless account-only tool asks the API for the caller\'s own link', async (t) => {
const backend = await startFakeFirecrawlBackend({
keylessEligibilityResponse: (url) => ({
status: 200,
body: url.includes('signup_link=1')
? { eligible: true, signupUrl: OWN_SIGNUP_URL }
: { eligible: true },
}),
});
t.after(() => backend.close());
const port = await getFreePort();
const child = spawnServer({
CLOUD_SERVICE: 'true',
FASTMCP_ENDPOINT: '/v2/mcp',
FIRECRAWL_API_URL: backend.url,
HTTP_STREAMABLE_SERVER: 'true',
KEYLESS_PROXY_SECRET: 'keyless-secret',
PORT: String(port),
});
t.after(() => stopChild(child));
await waitForHealth(port, child);

const response = await httpToolCall(port, {
id: 'keyless-account-only-own-link',
headers: { 'x-forwarded-for': '8.8.8.7' },
params: { arguments: { url: 'https://example.com/' }, name: 'firecrawl_crawl' },
});
const result = parseSseJson(await response.text()).result;
assertKeylessAccountRecovery(result, {
code: 'KEYLESS_TOOL_NOT_AVAILABLE',
message: keylessToolMessage(OWN_SIGNUP_URL),
});
const linkRequest = backend.requests.find((r) =>
r.url === '/v2/keyless/eligibility?signup_link=1'
);
assert.ok(linkRequest);
assert.equal(linkRequest.headers['x-firecrawl-keyless-ip'], '8.8.8.7');
assert.equal(linkRequest.headers['x-firecrawl-keyless-secret'], 'keyless-secret');
assert.equal(backend.requests.some((r) => r.url === '/v2/crawl'), false);
});

test('HTTP cloud keyless Parse completes both phases without credentials and forwards redactPII', async (t) => {
const backend = await startFakeFirecrawlBackend({ keylessEligible: true });
t.after(() => backend.close());
Expand Down
Loading