Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/__tests__/commands/parse.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ describe('executeParse', () => {
];
expect(url).toBe('https://api.firecrawl.dev/v2/parse');
expect(init.method).toBe('POST');
expect(init.headers).toBeUndefined();
// No Authorization; X-Origin attributes the keyless call to the CLI.
expect(init.headers).toEqual({ 'X-Origin': 'cli' });

const options = JSON.parse(init.body.get('options') as string);
expect(options).toEqual({
Expand Down
81 changes: 47 additions & 34 deletions src/__tests__/utils/client.test.ts
Original file line number Diff line number Diff line change
@@ -1,66 +1,79 @@
/**
* Tests for keyless request errors
*
* The API links every keyless prompt to signup tagged `utm_medium=api`. The CLI
* must retag that link as `cli` so signups started from the CLI are attributed
* to it.
* The API links every keyless prompt to the caller's own opaque signup link,
* https://firecrawl.dev/k/<id>, which the site resolves to CLI attribution when
* the request came from the CLI. The CLI prints that link unchanged and tells
* the API it is the CLI with X-Origin, including on requests without a body.
*/

import { describe, it, expect, vi, afterEach } from 'vitest';
import {
keylessGet,
keylessRequest,
withCliSignupTag,
} from '../../utils/client';
import { keylessGet, keylessRequest } from '../../utils/client';

const API_LIMIT_MESSAGE = `You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api
const OWN_SIGNUP_URL = 'https://firecrawl.dev/k/7fq2xab9';

const API_LIMIT_MESSAGE = `You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at ${OWN_SIGNUP_URL}

Then authenticate with:
Authorization: Bearer YOUR_API_KEY`;

const CLI_SIGNUP_URL =
'https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=cli';
// Before the /k links, the API sent a UTM-tagged link. An API still sending it
// must not be rewritten into something else.
const LEGACY_LIMIT_MESSAGE =
"You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api";

function stubFetch(status: number, body: unknown) {
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response(JSON.stringify(body), { status }))
const fetchMock = vi.fn(
async (_url: string, _init?: RequestInit) =>
new Response(JSON.stringify(body), { status })
);
vi.stubGlobal('fetch', fetchMock);
return fetchMock;
}

describe('withCliSignupTag', () => {
it('retags the keyless signup link as cli', () => {
const message = withCliSignupTag(API_LIMIT_MESSAGE);

expect(message).toContain(CLI_SIGNUP_URL);
expect(message).not.toContain('utm_medium=api');
});

it('leaves messages without the keyless signup link unchanged', () => {
expect(withCliSignupTag('Firecrawl request failed (HTTP 500)')).toBe(
'Firecrawl request failed (HTTP 500)'
);
});
});

describe('keyless requests', () => {
afterEach(() => {
vi.unstubAllGlobals();
});

it('reports the keyless limit with the cli signup link', async () => {
stubFetch(429, { success: false, error: API_LIMIT_MESSAGE });
it('reports the keyless limit with the API-issued signup link unchanged', async () => {
stubFetch(429, {
success: false,
error: API_LIMIT_MESSAGE,
signup_url: OWN_SIGNUP_URL,
});

await expect(
keylessRequest('/v2/scrape', { url: 'https://example.com' })
).rejects.toThrow(CLI_SIGNUP_URL);
).rejects.toThrow(API_LIMIT_MESSAGE);
});

it('reports the keyless limit on GET requests with the cli signup link', async () => {
it('reports the keyless limit on GET requests with the API-issued link', async () => {
stubFetch(429, { success: false, error: API_LIMIT_MESSAGE });

await expect(keylessGet('/v2/research/search?q=test')).rejects.toThrow(
CLI_SIGNUP_URL
OWN_SIGNUP_URL
);
});

it('no longer rewrites a legacy UTM link', async () => {
stubFetch(429, { success: false, error: LEGACY_LIMIT_MESSAGE });

await expect(
keylessRequest('/v2/scrape', { url: 'https://example.com' })
).rejects.toThrow(LEGACY_LIMIT_MESSAGE);
});

it('identifies the CLI with X-Origin on POST and GET requests', async () => {
const fetchMock = stubFetch(200, { success: true });

await keylessRequest('/v2/scrape', { url: 'https://example.com' });
await keylessGet('/v2/research/search?q=test');

for (const [, init] of fetchMock.mock.calls) {
const headers = init?.headers as Record<string, string>;
expect(headers['X-Origin']).toBe('cli');
expect(headers.Authorization).toBeUndefined();
}
});
});
15 changes: 6 additions & 9 deletions src/commands/interact.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Execute AI prompts or code against a scraped page in a live browser session
*/

import { getClient, isKeylessMode, withCliSignupTag } from '../utils/client';
import { getClient, isKeylessMode, KEYLESS_CLI_HEADERS } from '../utils/client';
import { getConfig, validateConfig } from '../utils/config';
import {
getScrapeId,
Expand Down Expand Up @@ -59,6 +59,7 @@ function buildHeaders(apiKey: string | undefined, keyless: boolean) {
if (!keyless && apiKey) {
headers.Authorization = `Bearer ${apiKey}`;
}
if (keyless) Object.assign(headers, KEYLESS_CLI_HEADERS);
return headers;
}

Expand Down Expand Up @@ -100,10 +101,8 @@ export async function handleInteractExecute(
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
throw new Error(
withCliSignupTag(
(errorData as any).error ||
`HTTP ${response.status}: ${response.statusText}`
)
(errorData as any).error ||
`HTTP ${response.status}: ${response.statusText}`
);
}

Expand Down Expand Up @@ -169,10 +168,8 @@ export async function handleInteractStop(
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
throw new Error(
withCliSignupTag(
(errorData as any).error ||
`HTTP ${response.status}: ${response.statusText}`
)
(errorData as any).error ||
`HTTP ${response.status}: ${response.statusText}`
);
}

Expand Down
15 changes: 10 additions & 5 deletions src/commands/parse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import * as path from 'path';
import type { FormatOption } from 'firecrawl';
import type { ParseOptions, ParseResult } from '../types/parse';
import type { ScrapeFormat } from '../types/scrape';
import { getClient, isKeylessMode, withCliSignupTag } from '../utils/client';
import { getClient, isKeylessMode, KEYLESS_CLI_HEADERS } from '../utils/client';
import { getConfig, validateConfig } from '../utils/config';
import { handleScrapeOutput } from '../utils/output';

Expand Down Expand Up @@ -184,8 +184,14 @@ export async function executeParse(
try {
const response = await fetch(`${apiUrl}/v2/parse`, {
method: 'POST',
// Multipart options are parsed after auth, so the header carries the
// CLI origin to the keyless check.
headers:
!keyless && apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined,
!keyless && apiKey
? { Authorization: `Bearer ${apiKey}` }
: keyless
? { ...KEYLESS_CLI_HEADERS }
: undefined,
body: form,
});

Expand All @@ -195,10 +201,9 @@ export async function executeParse(
const payload = (await response.json().catch(() => ({}))) as any;

if (!response.ok || payload?.success === false) {
const message = withCliSignupTag(
const message =
payload?.error ||
`HTTP ${response.status}: ${response.statusText || 'Request failed'}`
);
`HTTP ${response.status}: ${response.statusText || 'Request failed'}`;
return { success: false, error: message };
}

Expand Down
28 changes: 12 additions & 16 deletions src/utils/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@ export function isKeylessMode(apiKey?: string, apiUrl?: string): boolean {
}

/**
* The API's keyless prompts link to signup tagged `utm_medium=api`. Retag them
* as `cli` so accounts created from the CLI are attributed to the CLI.
* Headers for keyless requests. The API reads X-Origin to attribute keyless
* use, and a keyless prompt's signup link, to the CLI; requests without a body
* (GET research and developer lookups, interact stop) carry nothing else.
* Keyless error messages are printed as the API sends them: their
* firecrawl.dev/k/<id> link already resolves to CLI attribution.
*/
export function withCliSignupTag(message: string): string {
return message.replaceAll(
'utm_source=keyless&utm_medium=api',
'utm_source=keyless&utm_medium=cli'
);
}
export const KEYLESS_CLI_HEADERS: Readonly<Record<string, string>> = {
'X-Origin': 'cli',
};

export async function keylessRequest(
path: string,
Expand All @@ -47,15 +47,13 @@ export async function keylessRequest(
const apiUrl = (getConfig().apiUrl || DEFAULT_API_URL).replace(/\/$/, '');
const response = await fetch(`${apiUrl}${path}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: { 'Content-Type': 'application/json', ...KEYLESS_CLI_HEADERS },
body: JSON.stringify(body),
});
const json: any = await response.json().catch(() => ({}));
if (!response.ok) {
throw new Error(
withCliSignupTag(
json?.error || `Firecrawl request failed (HTTP ${response.status})`
)
json?.error || `Firecrawl request failed (HTTP ${response.status})`
);
}
return json;
Expand All @@ -65,14 +63,12 @@ export async function keylessGet(path: string): Promise<any> {
const apiUrl = (getConfig().apiUrl || DEFAULT_API_URL).replace(/\/$/, '');
const response = await fetch(`${apiUrl}${path}`, {
method: 'GET',
headers: { 'Content-Type': 'application/json' },
headers: { 'Content-Type': 'application/json', ...KEYLESS_CLI_HEADERS },
});
const json: any = await response.json().catch(() => ({}));
if (!response.ok) {
throw new Error(
withCliSignupTag(
json?.error || `Firecrawl request failed (HTTP ${response.status})`
)
json?.error || `Firecrawl request failed (HTTP ${response.status})`
);
}
return json;
Expand Down
Loading