From 2e8554bbe4ec118fcf569f332430db6a3b7f882c Mon Sep 17 00:00:00 2001 From: Zhongyue Lin Date: Fri, 31 Jul 2026 07:59:52 +0800 Subject: [PATCH] fix(xiaohongshu): retry once through a cooldown on risk-control soft blocks (#1825) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reading XHS note-detail pages (note / comments / download) back-to-back trips velocity-based risk control: a soft block that redirects to website-login/error?error_code=300017/300031 or renders "安全限制" / "访问链接异常" (#1825, #962). Today the first block fails the command outright, and an unattended loop keeps hammering — which escalates the risk state toward the account-violation / ban path (#842, #677). XHS soft blocks are frequently transient per-request challenges, so a single reload after a real cooldown recovers many of them. Add a shared `readXhsDetailPage` helper that navigates, settles, extracts, and — only on a security block — waits one long randomized cooldown (8–18s) and reloads exactly ONCE before surfacing SECURITY_BLOCK. The single-retry cap is structural (a guarded `if`, no caller-tunable retry count) so it can never devolve into a hammer loop; `retryOnBlock: false` opts into the previous fail-fast behavior. note / comments / download now share this helper, replacing three copies of the inline securityBlock detection. This does NOT throttle request velocity across separate CLI invocations — that needs session-level pacing and is left as a follow-up. Adds offline tests for the helper (transient-block recovery, the structural single-retry cap, fail-fast, custom settle bounds, non-block pass-through); the existing note/comments/download command tests pass unchanged. --- clis/xiaohongshu/comments.js | 21 ++--- clis/xiaohongshu/download.js | 21 +++-- clis/xiaohongshu/note.js | 21 ++--- clis/xiaohongshu/risk-control.js | 88 +++++++++++++++++++++ clis/xiaohongshu/risk-control.test.js | 109 ++++++++++++++++++++++++++ 5 files changed, 234 insertions(+), 26 deletions(-) create mode 100644 clis/xiaohongshu/risk-control.js create mode 100644 clis/xiaohongshu/risk-control.test.js diff --git a/clis/xiaohongshu/comments.js b/clis/xiaohongshu/comments.js index 0e9cefb4a..ac899f83a 100644 --- a/clis/xiaohongshu/comments.js +++ b/clis/xiaohongshu/comments.js @@ -6,8 +6,9 @@ * the --with-replies flag. */ import { cli, Strategy } from '@jackwener/opencli/registry'; -import { AuthRequiredError, CliError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors'; +import { AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors'; import { parseNoteId, buildNoteUrl } from './note-helpers.js'; +import { readXhsDetailPage } from './risk-control.js'; const XHS_PROFILE_HREF_SELECTOR = '.author-wrapper a[href*="/user/profile/"], a.name[href*="/user/profile/"], a.user-name[href*="/user/profile/"], a[href*="/user/profile/"]'; @@ -300,17 +301,19 @@ export const command = cli({ const withReplies = Boolean(kwargs['with-replies']); const raw = String(kwargs['note-id']); const noteId = parseNoteId(raw); - await page.goto(buildNoteUrl(raw, { commandName: 'xiaohongshu comments' })); - await page.wait({ time: 2 + Math.random() * 3 }); - const data = await page.evaluate(buildCommentsExtractJs(withReplies, limit)); + // readXhsDetailPage paces the navigation and retries once through a + // cooldown if risk control soft-blocks the page (throws SECURITY_BLOCK + // when still blocked after the retry). + const data = await readXhsDetailPage(page, { + url: buildNoteUrl(raw, { commandName: 'xiaohongshu comments' }), + extractJs: buildCommentsExtractJs(withReplies, limit), + securityHelp: /^https?:\/\//.test(raw) + ? 'The page may be temporarily restricted. Try again later or from a different session.' + : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.', + }); if (!data || typeof data !== 'object') { throw new EmptyResultError('xiaohongshu/comments', 'Unexpected evaluate response'); } - if (data.securityBlock) { - throw new CliError('SECURITY_BLOCK', 'Xiaohongshu security block: the note detail page was blocked by risk control.', /^https?:\/\//.test(raw) - ? 'The page may be temporarily restricted. Try again later or from a different session.' - : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.'); - } if (data.loginWall) { throw new AuthRequiredError('www.xiaohongshu.com', 'Note comments require login'); } diff --git a/clis/xiaohongshu/download.js b/clis/xiaohongshu/download.js index a9df7fa0c..849f40d1d 100644 --- a/clis/xiaohongshu/download.js +++ b/clis/xiaohongshu/download.js @@ -9,7 +9,8 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { formatCookieHeader } from '@jackwener/opencli/download'; import { downloadMedia } from '@jackwener/opencli/download/media-download'; -import { CliError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors'; +import { CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors'; +import { readXhsDetailPage } from './risk-control.js'; import { buildNoteUrl, parseNoteId } from './note-helpers.js'; /** * Build the media-extraction IIFE. The note id is interpolated as a default @@ -219,14 +220,18 @@ export const command = cli({ const rawInput = String(kwargs['note-id']); const output = kwargs.output; const noteId = parseNoteId(rawInput); - await page.goto(buildNoteUrl(rawInput, { allowShortLink: true, commandName: 'xiaohongshu download' })); - await page.wait({ time: 1 + Math.random() * 2 }); - const data = await page.evaluate(buildDownloadExtractJs(noteId)); - if (data?.securityBlock) { - throw new CliError('SECURITY_BLOCK', 'Xiaohongshu security block: the note detail page was blocked by risk control.', /^https?:\/\//.test(rawInput) + // readXhsDetailPage paces the navigation and retries once through a + // cooldown if risk control soft-blocks the page (throws SECURITY_BLOCK + // when still blocked after the retry). + const data = await readXhsDetailPage(page, { + url: buildNoteUrl(rawInput, { allowShortLink: true, commandName: 'xiaohongshu download' }), + extractJs: buildDownloadExtractJs(noteId), + securityHelp: /^https?:\/\//.test(rawInput) ? 'The page may be temporarily restricted. Try again later or from a different session.' - : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.'); - } + : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.', + settleMinS: 1, + settleMaxS: 3, + }); if (!data || typeof data !== 'object' || !Array.isArray(data.media)) { throw new CommandExecutionError('Xiaohongshu media extraction returned malformed payload.'); } diff --git a/clis/xiaohongshu/note.js b/clis/xiaohongshu/note.js index 17f4d2687..50e2e9f8a 100644 --- a/clis/xiaohongshu/note.js +++ b/clis/xiaohongshu/note.js @@ -7,8 +7,9 @@ * Requires a full Xiaohongshu note URL with xsec_token. */ import { cli, Strategy } from '@jackwener/opencli/registry'; -import { AuthRequiredError, CliError, EmptyResultError } from '@jackwener/opencli/errors'; +import { AuthRequiredError, EmptyResultError } from '@jackwener/opencli/errors'; import { parseNoteId, buildNoteUrl } from './note-helpers.js'; +import { readXhsDetailPage } from './risk-control.js'; /** * Host-agnostic IIFE that scrapes note title / author / counts / tags from a * rendered note detail page. Exported so the rednote adapter can reuse the @@ -62,17 +63,19 @@ export const command = cli({ const raw = String(kwargs['note-id']); const noteId = parseNoteId(raw); const url = buildNoteUrl(raw, { commandName: 'xiaohongshu note' }); - await page.goto(url); - await page.wait({ time: 2 + Math.random() * 3 }); - const data = await page.evaluate(NOTE_EXTRACT_JS); + // readXhsDetailPage paces the navigation and retries once through a + // cooldown if risk control soft-blocks the page (throws SECURITY_BLOCK + // when still blocked after the retry). + const data = await readXhsDetailPage(page, { + url, + extractJs: NOTE_EXTRACT_JS, + securityHelp: /^https?:\/\//.test(raw) + ? 'The page may be temporarily restricted. Try again later or from a different session.' + : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.', + }); if (!data || typeof data !== 'object') { throw new EmptyResultError('xiaohongshu/note', 'Unexpected evaluate response'); } - if (data.securityBlock) { - throw new CliError('SECURITY_BLOCK', 'Xiaohongshu security block: the note detail page was blocked by risk control.', /^https?:\/\//.test(raw) - ? 'The page may be temporarily restricted. Try again later or from a different session.' - : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.'); - } if (data.loginWall) { throw new AuthRequiredError('www.xiaohongshu.com', 'Note content requires login'); } diff --git a/clis/xiaohongshu/risk-control.js b/clis/xiaohongshu/risk-control.js new file mode 100644 index 000000000..b174982b5 --- /dev/null +++ b/clis/xiaohongshu/risk-control.js @@ -0,0 +1,88 @@ +import { CliError } from '@jackwener/opencli/errors'; + +/** + * Xiaohongshu risk-control pacing shared by the note / comments / download + * detail-page commands. + * + * XHS gates note-detail navigation behind velocity-based risk control: reading a + * run of notes back-to-back trips a soft block that redirects to + * `website-login/error?error_code=300017` / `300031` or renders "安全限制" / + * "访问链接异常" (issues #1825, #962). Those soft blocks are frequently transient + * per-request challenges — a single reload after a real cooldown clears many of + * them. So instead of failing on the first block, retry ONCE after a long + * randomized cooldown. + * + * The retry is deliberately capped at one: hammering a hot risk state is exactly + * what escalates it toward the account-violation / ban path (#842, #677). This + * helper only makes each read gentler and recovers transient blocks — it does + * NOT cap request velocity across separate CLI invocations (that needs + * session-level throttling, tracked as a follow-up). + */ + +/** Randomized delay in seconds within [minS, maxS]. `rand` is injectable for tests. */ +export function jitterSeconds(minS, maxS, rand = Math.random) { + return minS + rand() * (maxS - minS); +} + +/** A detail-page extract payload signals risk control via `securityBlock: true`. */ +export function isSecurityBlock(data) { + return Boolean(data && typeof data === 'object' && !Array.isArray(data) && data.securityBlock); +} + +/** + * Navigate to a XHS detail page and run `extractJs`, retrying once through a long + * randomized cooldown when risk control soft-blocks the page. Returns the extract + * payload (never a security-block payload — that path throws SECURITY_BLOCK after + * the single retry is exhausted). Callers keep their own loginWall / notFound / + * shape handling on the returned payload. + * + * @param {object} page Browser Bridge page handle. + * @param {object} opts + * @param {string} opts.url Fully-built note/detail URL to navigate to. + * @param {string} opts.extractJs Page-side extraction IIFE returning `{ securityBlock, ... }`. + * @param {string} [opts.securityHelp] Hint attached to the thrown SECURITY_BLOCK error. + * @param {number} [opts.settleMinS] Min settle delay after navigation (seconds). + * @param {number} [opts.settleMaxS] Max settle delay after navigation (seconds). + * @param {boolean} [opts.retryOnBlock] Do the single cooldown reload on a soft block (default true); false = fail fast. + * @param {number} [opts.cooldownMinS] Min cooldown before the retry (seconds). + * @param {number} [opts.cooldownMaxS] Max cooldown before the retry (seconds). + * @param {() => number} [opts.rand] Injectable RNG for deterministic tests. + */ +export async function readXhsDetailPage(page, { + url, + extractJs, + securityHelp, + settleMinS = 2, + settleMaxS = 5, + retryOnBlock = true, + cooldownMinS = 8, + cooldownMaxS = 18, + rand = Math.random, +} = {}) { + const readOnce = async () => { + await page.goto(url); + await page.wait({ time: jitterSeconds(settleMinS, settleMaxS, rand) }); + return page.evaluate(extractJs); + }; + + let data = await readOnce(); + // At most ONE retry — a single `if`, never a loop. Hammering a hot risk state + // is exactly what escalates it toward account-violation / ban (#842, #677), + // so the one-cooldown-reload cap is enforced structurally, not by a caller's + // choice of retry count. + if (retryOnBlock && isSecurityBlock(data)) { + await page.wait({ time: jitterSeconds(cooldownMinS, cooldownMaxS, rand) }); + data = await readOnce(); + } + + if (isSecurityBlock(data)) { + throw new CliError( + 'SECURITY_BLOCK', + 'Xiaohongshu security block: the note detail page was blocked by risk control.', + securityHelp, + ); + } + return data; +} + +export const __test__ = { jitterSeconds, isSecurityBlock, readXhsDetailPage }; diff --git a/clis/xiaohongshu/risk-control.test.js b/clis/xiaohongshu/risk-control.test.js new file mode 100644 index 000000000..616d5d1bf --- /dev/null +++ b/clis/xiaohongshu/risk-control.test.js @@ -0,0 +1,109 @@ +import { describe, expect, it, vi } from 'vitest'; +import { CliError } from '@jackwener/opencli/errors'; +import { __test__ } from './risk-control.js'; + +const { jitterSeconds, isSecurityBlock, readXhsDetailPage } = __test__; + +function makePage(evaluateResults) { + let i = 0; + return { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn().mockImplementation(() => + Promise.resolve(evaluateResults[Math.min(i++, evaluateResults.length - 1)])), + }; +} + +describe('xiaohongshu risk-control jitterSeconds', () => { + it('stays within [min, max] and tracks rand', () => { + expect(jitterSeconds(2, 5, () => 0)).toBe(2); + expect(jitterSeconds(2, 5, () => 1)).toBe(5); + expect(jitterSeconds(2, 5, () => 0.5)).toBe(3.5); + const v = jitterSeconds(8, 18); // real Math.random + expect(v).toBeGreaterThanOrEqual(8); + expect(v).toBeLessThanOrEqual(18); + }); +}); + +describe('xiaohongshu risk-control isSecurityBlock', () => { + it('is true only for a plain object flagged securityBlock', () => { + expect(isSecurityBlock({ securityBlock: true })).toBe(true); + expect(isSecurityBlock({ securityBlock: false })).toBe(false); + expect(isSecurityBlock({})).toBe(false); + expect(isSecurityBlock(null)).toBe(false); + expect(isSecurityBlock(undefined)).toBe(false); + expect(isSecurityBlock([{ securityBlock: true }])).toBe(false); // arrays are not payloads + expect(isSecurityBlock('securityBlock')).toBe(false); + }); +}); + +describe('xiaohongshu risk-control readXhsDetailPage', () => { + const url = 'https://www.xiaohongshu.com/search_result/abc?xsec_token=tok'; + const extractJs = '(() => ({}))()'; + + it('returns the payload on first read without any cooldown when not blocked', async () => { + const page = makePage([{ title: 'ok', securityBlock: false }]); + const data = await readXhsDetailPage(page, { url, extractJs, rand: () => 0.5 }); + expect(data).toEqual({ title: 'ok', securityBlock: false }); + expect(page.goto).toHaveBeenCalledTimes(1); + expect(page.evaluate).toHaveBeenCalledTimes(1); + // only the settle wait, no cooldown + expect(page.wait).toHaveBeenCalledTimes(1); + }); + + it('recovers a transient soft-block with a single cooldown retry', async () => { + const page = makePage([{ securityBlock: true }, { title: 'recovered', securityBlock: false }]); + const data = await readXhsDetailPage(page, { url, extractJs, rand: () => 0.5 }); + expect(data).toEqual({ title: 'recovered', securityBlock: false }); + // re-navigated + re-extracted exactly once more + expect(page.goto).toHaveBeenCalledTimes(2); + expect(page.evaluate).toHaveBeenCalledTimes(2); + // a long cooldown wait happened between the two reads: 8 + 0.5*(18-8) = 13 + expect(page.wait).toHaveBeenCalledWith({ time: 13 }); + }); + + it('throws SECURITY_BLOCK (with the hint) when still blocked after the one retry — never hammers', async () => { + const page = makePage([{ securityBlock: true }, { securityBlock: true }, { securityBlock: true }]); + await expect(readXhsDetailPage(page, { + url, + extractJs, + securityHelp: 'Try again later or from a different session.', + rand: () => 0.5, + })).rejects.toMatchObject({ + code: 'SECURITY_BLOCK', + hint: 'Try again later or from a different session.', + }); + // exactly one retry — goto/evaluate called twice, not more + expect(page.goto).toHaveBeenCalledTimes(2); + expect(page.evaluate).toHaveBeenCalledTimes(2); + }); + + it('fails fast without a retry when retryOnBlock is false', async () => { + const page = makePage([{ securityBlock: true }, { title: 'never reached', securityBlock: false }]); + await expect(readXhsDetailPage(page, { url, extractJs, retryOnBlock: false, rand: () => 0.5 })) + .rejects.toBeInstanceOf(CliError); + // no cooldown reload — exactly one navigation/extraction + expect(page.goto).toHaveBeenCalledTimes(1); + expect(page.evaluate).toHaveBeenCalledTimes(1); + }); + + it('respects custom settle bounds (download uses 1-3s)', async () => { + const page = makePage([{ media: [], securityBlock: false }]); + await readXhsDetailPage(page, { url, extractJs, settleMinS: 1, settleMaxS: 3, rand: () => 0 }); + // settle = 1 + 0*(3-1) = 1 + expect(page.wait).toHaveBeenCalledWith({ time: 1 }); + }); + + it('passes non-block malformed payloads straight through (caller handles them)', async () => { + const page = makePage([null]); + const data = await readXhsDetailPage(page, { url, extractJs, rand: () => 0.5 }); + expect(data).toBeNull(); + expect(page.goto).toHaveBeenCalledTimes(1); // no retry for a non-block result + }); + + it('surfaces SECURITY_BLOCK as a CliError instance', async () => { + const page = makePage([{ securityBlock: true }, { securityBlock: true }]); + await expect(readXhsDetailPage(page, { url, extractJs, rand: () => 0.5 })) + .rejects.toBeInstanceOf(CliError); + }); +});