fix(xiaohongshu): retry once through a cooldown on risk-control soft blocks (#1825)

clis/xiaohongshu/comments.js

@@ -6,8 +6,9 @@
  * the --with-replies flag.
  */
 import { cli, Strategy } from '@jackwener/opencli/registry';
-import { AuthRequiredError, CliError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
+import { AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
 import { parseNoteId, buildNoteUrl } from './note-helpers.js';
+import { readXhsDetailPage } from './risk-control.js';
 
 const XHS_PROFILE_HREF_SELECTOR = '.author-wrapper a[href*="/user/profile/"], a.name[href*="/user/profile/"], a.user-name[href*="/user/profile/"], a[href*="/user/profile/"]';
 
@@ -300,17 +301,19 @@ export const command = cli({
         const withReplies = Boolean(kwargs['with-replies']);
         const raw = String(kwargs['note-id']);
         const noteId = parseNoteId(raw);
-        await page.goto(buildNoteUrl(raw, { commandName: 'xiaohongshu comments' }));
-        await page.wait({ time: 2 + Math.random() * 3 });
-        const data = await page.evaluate(buildCommentsExtractJs(withReplies, limit));
+        // readXhsDetailPage paces the navigation and retries once through a
+        // cooldown if risk control soft-blocks the page (throws SECURITY_BLOCK
+        // when still blocked after the retry).
+        const data = await readXhsDetailPage(page, {
+            url: buildNoteUrl(raw, { commandName: 'xiaohongshu comments' }),
+            extractJs: buildCommentsExtractJs(withReplies, limit),
+            securityHelp: /^https?:\/\//.test(raw)
+                ? 'The page may be temporarily restricted. Try again later or from a different session.'
+                : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.',
+        });
         if (!data || typeof data !== 'object') {
             throw new EmptyResultError('xiaohongshu/comments', 'Unexpected evaluate response');
         }
-        if (data.securityBlock) {
-            throw new CliError('SECURITY_BLOCK', 'Xiaohongshu security block: the note detail page was blocked by risk control.', /^https?:\/\//.test(raw)
-                ? 'The page may be temporarily restricted. Try again later or from a different session.'
-                : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.');
-        }
         if (data.loginWall) {
             throw new AuthRequiredError('www.xiaohongshu.com', 'Note comments require login');
         }

clis/xiaohongshu/download.js

@@ -9,7 +9,8 @@
 import { cli, Strategy } from '@jackwener/opencli/registry';
 import { formatCookieHeader } from '@jackwener/opencli/download';
 import { downloadMedia } from '@jackwener/opencli/download/media-download';
-import { CliError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
+import { CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
+import { readXhsDetailPage } from './risk-control.js';
 import { buildNoteUrl, parseNoteId } from './note-helpers.js';
 /**
  * Build the media-extraction IIFE. The note id is interpolated as a default
@@ -219,14 +220,18 @@ export const command = cli({
         const rawInput = String(kwargs['note-id']);
         const output = kwargs.output;
         const noteId = parseNoteId(rawInput);
-        await page.goto(buildNoteUrl(rawInput, { allowShortLink: true, commandName: 'xiaohongshu download' }));
-        await page.wait({ time: 1 + Math.random() * 2 });
-        const data = await page.evaluate(buildDownloadExtractJs(noteId));
-        if (data?.securityBlock) {
-            throw new CliError('SECURITY_BLOCK', 'Xiaohongshu security block: the note detail page was blocked by risk control.', /^https?:\/\//.test(rawInput)
+        // readXhsDetailPage paces the navigation and retries once through a
+        // cooldown if risk control soft-blocks the page (throws SECURITY_BLOCK
+        // when still blocked after the retry).
+        const data = await readXhsDetailPage(page, {
+            url: buildNoteUrl(rawInput, { allowShortLink: true, commandName: 'xiaohongshu download' }),
+            extractJs: buildDownloadExtractJs(noteId),
+            securityHelp: /^https?:\/\//.test(rawInput)
                 ? 'The page may be temporarily restricted. Try again later or from a different session.'
-                : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.');
-        }
+                : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.',
+            settleMinS: 1,
+            settleMaxS: 3,
+        });
         if (!data || typeof data !== 'object' || !Array.isArray(data.media)) {
             throw new CommandExecutionError('Xiaohongshu media extraction returned malformed payload.');
         }

clis/xiaohongshu/note.js

@@ -7,8 +7,9 @@
  * Requires a full Xiaohongshu note URL with xsec_token.
  */
 import { cli, Strategy } from '@jackwener/opencli/registry';
-import { AuthRequiredError, CliError, EmptyResultError } from '@jackwener/opencli/errors';
+import { AuthRequiredError, EmptyResultError } from '@jackwener/opencli/errors';
 import { parseNoteId, buildNoteUrl } from './note-helpers.js';
+import { readXhsDetailPage } from './risk-control.js';
 /**
  * Host-agnostic IIFE that scrapes note title / author / counts / tags from a
  * rendered note detail page. Exported so the rednote adapter can reuse the
@@ -62,17 +63,19 @@ export const command = cli({
         const raw = String(kwargs['note-id']);
         const noteId = parseNoteId(raw);
         const url = buildNoteUrl(raw, { commandName: 'xiaohongshu note' });
-        await page.goto(url);
-        await page.wait({ time: 2 + Math.random() * 3 });
-        const data = await page.evaluate(NOTE_EXTRACT_JS);
+        // readXhsDetailPage paces the navigation and retries once through a
+        // cooldown if risk control soft-blocks the page (throws SECURITY_BLOCK
+        // when still blocked after the retry).
+        const data = await readXhsDetailPage(page, {
+            url,
+            extractJs: NOTE_EXTRACT_JS,
+            securityHelp: /^https?:\/\//.test(raw)
+                ? 'The page may be temporarily restricted. Try again later or from a different session.'
+                : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.',
+        });
         if (!data || typeof data !== 'object') {
             throw new EmptyResultError('xiaohongshu/note', 'Unexpected evaluate response');
         }
-        if (data.securityBlock) {
-            throw new CliError('SECURITY_BLOCK', 'Xiaohongshu security block: the note detail page was blocked by risk control.', /^https?:\/\//.test(raw)
-                ? 'The page may be temporarily restricted. Try again later or from a different session.'
-                : 'Try using a full URL from search results (with xsec_token) instead of a bare note ID.');
-        }
         if (data.loginWall) {
             throw new AuthRequiredError('www.xiaohongshu.com', 'Note content requires login');
         }

clis/xiaohongshu/risk-control.js

@@ -0,0 +1,88 @@
+import { CliError } from '@jackwener/opencli/errors';
+
+/**
+ * Xiaohongshu risk-control pacing shared by the note / comments / download
+ * detail-page commands.
+ *
+ * XHS gates note-detail navigation behind velocity-based risk control: reading a
+ * run of notes back-to-back trips a soft block that redirects to
+ * `website-login/error?error_code=300017` / `300031` or renders "安全限制" /
+ * "访问链接异常" (issues #1825, #962). Those soft blocks are frequently transient
+ * per-request challenges — a single reload after a real cooldown clears many of
+ * them. So instead of failing on the first block, retry ONCE after a long
+ * randomized cooldown.
+ *
+ * The retry is deliberately capped at one: hammering a hot risk state is exactly
+ * what escalates it toward the account-violation / ban path (#842, #677). This
+ * helper only makes each read gentler and recovers transient blocks — it does
+ * NOT cap request velocity across separate CLI invocations (that needs
+ * session-level throttling, tracked as a follow-up).
+ */
+
+/** Randomized delay in seconds within [minS, maxS]. `rand` is injectable for tests. */
+export function jitterSeconds(minS, maxS, rand = Math.random) {
+    return minS + rand() * (maxS - minS);
+}
+
+/** A detail-page extract payload signals risk control via `securityBlock: true`. */
+export function isSecurityBlock(data) {
+    return Boolean(data && typeof data === 'object' && !Array.isArray(data) && data.securityBlock);
+}
+
+/**
+ * Navigate to a XHS detail page and run `extractJs`, retrying once through a long
+ * randomized cooldown when risk control soft-blocks the page. Returns the extract
+ * payload (never a security-block payload — that path throws SECURITY_BLOCK after
+ * the single retry is exhausted). Callers keep their own loginWall / notFound /
+ * shape handling on the returned payload.
+ *
+ * @param {object} page Browser Bridge page handle.
+ * @param {object} opts
+ * @param {string} opts.url            Fully-built note/detail URL to navigate to.
+ * @param {string} opts.extractJs      Page-side extraction IIFE returning `{ securityBlock, ... }`.
+ * @param {string} [opts.securityHelp] Hint attached to the thrown SECURITY_BLOCK error.
+ * @param {number} [opts.settleMinS]   Min settle delay after navigation (seconds).
+ * @param {number} [opts.settleMaxS]   Max settle delay after navigation (seconds).
+ * @param {boolean} [opts.retryOnBlock] Do the single cooldown reload on a soft block (default true); false = fail fast.
+ * @param {number} [opts.cooldownMinS] Min cooldown before the retry (seconds).
+ * @param {number} [opts.cooldownMaxS] Max cooldown before the retry (seconds).
+ * @param {() => number} [opts.rand]   Injectable RNG for deterministic tests.
+ */
+export async function readXhsDetailPage(page, {
+    url,
+    extractJs,
+    securityHelp,
+    settleMinS = 2,
+    settleMaxS = 5,
+    retryOnBlock = true,
+    cooldownMinS = 8,
+    cooldownMaxS = 18,
+    rand = Math.random,
+} = {}) {
+    const readOnce = async () => {
+        await page.goto(url);
+        await page.wait({ time: jitterSeconds(settleMinS, settleMaxS, rand) });
+        return page.evaluate(extractJs);
+    };
+
+    let data = await readOnce();
+    // At most ONE retry — a single `if`, never a loop. Hammering a hot risk state
+    // is exactly what escalates it toward account-violation / ban (#842, #677),
+    // so the one-cooldown-reload cap is enforced structurally, not by a caller's
+    // choice of retry count.
+    if (retryOnBlock && isSecurityBlock(data)) {
+        await page.wait({ time: jitterSeconds(cooldownMinS, cooldownMaxS, rand) });
+        data = await readOnce();
+    }
+
+    if (isSecurityBlock(data)) {
+        throw new CliError(
+            'SECURITY_BLOCK',
+            'Xiaohongshu security block: the note detail page was blocked by risk control.',
+            securityHelp,
+        );
+    }
+    return data;
+}
+
+export const __test__ = { jitterSeconds, isSecurityBlock, readXhsDetailPage };

clis/xiaohongshu/risk-control.test.js

@@ -0,0 +1,109 @@
+import { describe, expect, it, vi } from 'vitest';
+import { CliError } from '@jackwener/opencli/errors';
+import { __test__ } from './risk-control.js';
+
+const { jitterSeconds, isSecurityBlock, readXhsDetailPage } = __test__;
+
+function makePage(evaluateResults) {
+    let i = 0;
+    return {
+        goto: vi.fn().mockResolvedValue(undefined),
+        wait: vi.fn().mockResolvedValue(undefined),
+        evaluate: vi.fn().mockImplementation(() =>
+            Promise.resolve(evaluateResults[Math.min(i++, evaluateResults.length - 1)])),
+    };
+}
+
+describe('xiaohongshu risk-control jitterSeconds', () => {
+    it('stays within [min, max] and tracks rand', () => {
+        expect(jitterSeconds(2, 5, () => 0)).toBe(2);
+        expect(jitterSeconds(2, 5, () => 1)).toBe(5);
+        expect(jitterSeconds(2, 5, () => 0.5)).toBe(3.5);
+        const v = jitterSeconds(8, 18); // real Math.random
+        expect(v).toBeGreaterThanOrEqual(8);
+        expect(v).toBeLessThanOrEqual(18);
+    });
+});
+
+describe('xiaohongshu risk-control isSecurityBlock', () => {
+    it('is true only for a plain object flagged securityBlock', () => {
+        expect(isSecurityBlock({ securityBlock: true })).toBe(true);
+        expect(isSecurityBlock({ securityBlock: false })).toBe(false);
+        expect(isSecurityBlock({})).toBe(false);
+        expect(isSecurityBlock(null)).toBe(false);
+        expect(isSecurityBlock(undefined)).toBe(false);
+        expect(isSecurityBlock([{ securityBlock: true }])).toBe(false); // arrays are not payloads
+        expect(isSecurityBlock('securityBlock')).toBe(false);
+    });
+});
+
+describe('xiaohongshu risk-control readXhsDetailPage', () => {
+    const url = 'https://www.xiaohongshu.com/search_result/abc?xsec_token=tok';
+    const extractJs = '(() => ({}))()';
+
+    it('returns the payload on first read without any cooldown when not blocked', async () => {
+        const page = makePage([{ title: 'ok', securityBlock: false }]);
+        const data = await readXhsDetailPage(page, { url, extractJs, rand: () => 0.5 });
+        expect(data).toEqual({ title: 'ok', securityBlock: false });
+        expect(page.goto).toHaveBeenCalledTimes(1);
+        expect(page.evaluate).toHaveBeenCalledTimes(1);
+        // only the settle wait, no cooldown
+        expect(page.wait).toHaveBeenCalledTimes(1);
+    });
+
+    it('recovers a transient soft-block with a single cooldown retry', async () => {
+        const page = makePage([{ securityBlock: true }, { title: 'recovered', securityBlock: false }]);
+        const data = await readXhsDetailPage(page, { url, extractJs, rand: () => 0.5 });
+        expect(data).toEqual({ title: 'recovered', securityBlock: false });
+        // re-navigated + re-extracted exactly once more
+        expect(page.goto).toHaveBeenCalledTimes(2);
+        expect(page.evaluate).toHaveBeenCalledTimes(2);
+        // a long cooldown wait happened between the two reads: 8 + 0.5*(18-8) = 13
+        expect(page.wait).toHaveBeenCalledWith({ time: 13 });
+    });
+
+    it('throws SECURITY_BLOCK (with the hint) when still blocked after the one retry — never hammers', async () => {
+        const page = makePage([{ securityBlock: true }, { securityBlock: true }, { securityBlock: true }]);
+        await expect(readXhsDetailPage(page, {
+            url,
+            extractJs,
+            securityHelp: 'Try again later or from a different session.',
+            rand: () => 0.5,
+        })).rejects.toMatchObject({
+            code: 'SECURITY_BLOCK',
+            hint: 'Try again later or from a different session.',
+        });
+        // exactly one retry — goto/evaluate called twice, not more
+        expect(page.goto).toHaveBeenCalledTimes(2);
+        expect(page.evaluate).toHaveBeenCalledTimes(2);
+    });
+
+    it('fails fast without a retry when retryOnBlock is false', async () => {
+        const page = makePage([{ securityBlock: true }, { title: 'never reached', securityBlock: false }]);
+        await expect(readXhsDetailPage(page, { url, extractJs, retryOnBlock: false, rand: () => 0.5 }))
+            .rejects.toBeInstanceOf(CliError);
+        // no cooldown reload — exactly one navigation/extraction
+        expect(page.goto).toHaveBeenCalledTimes(1);
+        expect(page.evaluate).toHaveBeenCalledTimes(1);
+    });
+
+    it('respects custom settle bounds (download uses 1-3s)', async () => {
+        const page = makePage([{ media: [], securityBlock: false }]);
+        await readXhsDetailPage(page, { url, extractJs, settleMinS: 1, settleMaxS: 3, rand: () => 0 });
+        // settle = 1 + 0*(3-1) = 1
+        expect(page.wait).toHaveBeenCalledWith({ time: 1 });
+    });
+
+    it('passes non-block malformed payloads straight through (caller handles them)', async () => {
+        const page = makePage([null]);
+        const data = await readXhsDetailPage(page, { url, extractJs, rand: () => 0.5 });
+        expect(data).toBeNull();
+        expect(page.goto).toHaveBeenCalledTimes(1); // no retry for a non-block result
+    });
+
+    it('surfaces SECURITY_BLOCK as a CliError instance', async () => {
+        const page = makePage([{ securityBlock: true }, { securityBlock: true }]);
+        await expect(readXhsDetailPage(page, { url, extractJs, rand: () => 0.5 }))
+            .rejects.toBeInstanceOf(CliError);
+    });
+});