From 2b15016badd10dd4b1c1593365ebf3bf18733ad5 Mon Sep 17 00:00:00 2001 From: Benjamin Liu Date: Wed, 5 Aug 2026 17:42:16 +0900 Subject: [PATCH] fix(douyin): search hashtags through the live endpoint and report empty responses --- clis/douyin/_shared/browser-fetch.js | 9 +++- clis/douyin/_shared/browser-fetch.test.js | 44 ++++++++++++++++ clis/douyin/hashtag.js | 24 +++++---- clis/douyin/hashtag.test.js | 63 ++++++++++++++++++++--- 4 files changed, 123 insertions(+), 17 deletions(-) diff --git a/clis/douyin/_shared/browser-fetch.js b/clis/douyin/_shared/browser-fetch.js index bece70871..e7b7a2bdc 100644 --- a/clis/douyin/_shared/browser-fetch.js +++ b/clis/douyin/_shared/browser-fetch.js @@ -27,6 +27,10 @@ export async function browserFetch(page, method, url, options = {}) { ${options.body ? `body: JSON.stringify(${JSON.stringify(options.body)}),` : ''} }); const text = await res.text(); + // A retired or gated endpoint answers 200 with no body. Reporting that + // as a parse failure sends readers after the JSON instead of the + // endpoint (issue #1405 fixed it, #1587 dropped it again). + if (!text.trim()) return res.ok ? null : { status_code: res.status, status_msg: 'Empty response body' }; try { return JSON.parse(text); } catch (error) { @@ -47,7 +51,10 @@ export async function browserFetch(page, method, url, options = {}) { throw new CommandExecutionError(`Douyin API request failed (${method} ${url}): ${error instanceof Error ? error.message : String(error)}`); } if (result == null) { - throw new CommandExecutionError(`Empty response from Douyin API (${method} ${url})`); + throw new CommandExecutionError( + `Empty response from Douyin API (${method} ${url})`, + 'The endpoint may have been retired or may now require signed parameters.', + ); } if (Array.isArray(result) || typeof result !== 'object') { throw new CommandExecutionError(`Malformed response from Douyin API (${method} ${url})`); diff --git a/clis/douyin/_shared/browser-fetch.test.js b/clis/douyin/_shared/browser-fetch.test.js index dd8c0233b..436444326 100644 --- a/clis/douyin/_shared/browser-fetch.test.js +++ b/clis/douyin/_shared/browser-fetch.test.js @@ -13,6 +13,15 @@ function makePage(result) { screenshot: vi.fn(), }; } +function makeScriptPage(body, { ok = true, status = 200, envelope = false } = {}) { + const page = makePage(null); + page.evaluate = vi.fn(async (script) => { + const stubFetch = async () => ({ ok, status, text: async () => body }); + const value = await new Function('fetch', `return (${script});`)(stubFetch); + return envelope ? { session: 'site:douyin:test', data: value } : value; + }); + return page; +} describe('browserFetch', () => { it('returns parsed JSON on success', async () => { const page = makePage({ status_code: 0, data: { ak: 'KEY' } }); @@ -38,6 +47,41 @@ describe('browserFetch', () => { const result = await browserFetch(page, 'GET', 'https://creator.douyin.com/api/test'); expect(result).toEqual({ some_field: 'value' }); }); + it('reports an empty body as an empty response, not a parse failure', async () => { + const page = makeScriptPage(''); + await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')) + .rejects.toThrow('Empty response from Douyin API'); + }); + it('treats a whitespace-only body the same way', async () => { + const page = makeScriptPage(' \n '); + await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')) + .rejects.toThrow('Empty response from Douyin API'); + }); + it('reports an empty body through a Browser Bridge {session,data} envelope', async () => { + const page = makeScriptPage('', { envelope: true }); + await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')) + .rejects.toThrow('Empty response from Douyin API'); + }); + it('still reports a non-JSON body as a parse failure', async () => { + const page = makeScriptPage('gateway'); + await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')) + .rejects.toThrow('JSON parse failed: gateway'); + }); + it('parses a JSON body from the generated script', async () => { + const page = makeScriptPage('{"status_code":0,"challenge_list":[]}'); + await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')) + .resolves.toEqual({ status_code: 0, challenge_list: [] }); + }); + it('keeps the auth classification when an empty body comes with 403', async () => { + const page = makeScriptPage('', { ok: false, status: 403 }); + await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')) + .rejects.toBeInstanceOf(AuthRequiredError); + }); + it('keeps the status when an empty body comes with 404', async () => { + const page = makeScriptPage('', { ok: false, status: 404 }); + await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')) + .rejects.toThrow('Douyin API error 404'); + }); it('throws on empty response body (null from evaluate)', async () => { const page = makePage(null); await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')).rejects.toThrow('Empty response from Douyin API'); diff --git a/clis/douyin/hashtag.js b/clis/douyin/hashtag.js index 743580f14..ee942374b 100644 --- a/clis/douyin/hashtag.js +++ b/clis/douyin/hashtag.js @@ -20,6 +20,10 @@ function requireListField(res, field, action) { function validateHashtagArgs(kwargs) { const action = kwargs.action; + const limit = Number(kwargs.limit ?? 10); + if (!Number.isInteger(limit) || limit < 1) { + throw new ArgumentError(`--limit must be a positive integer, got ${JSON.stringify(kwargs.limit)}`); + } if (action === 'search') { const keyword = String(kwargs.keyword ?? '').trim(); if (!keyword) { @@ -55,22 +59,24 @@ cli({ const action = kwargs.action; if (action === 'search') { const keyword = String(kwargs.keyword ?? '').trim(); - const url = `https://creator.douyin.com/aweme/v1/challenge/search/?keyword=${encodeURIComponent(keyword)}&count=${kwargs.limit}&aid=1128`; + // challenge/search answers 200 with an empty body; the creator + // studio composer reads suggestions from this endpoint instead, + // which ignores count and returns a fixed-size list (#2205). + const url = `https://creator.douyin.com/aweme/v1/search/challengesug/?keyword=${encodeURIComponent(keyword)}&source=challenge_create&aid=2906`; const res = await browserFetch(page, 'GET', url); - const list = requireListField(res, 'challenge_list', 'search'); + const list = requireListField(res, 'sug_list', 'search'); const rows = list.flatMap(c => { - const info = c?.challenge_info; - if (!isPlainObject(info)) return []; + if (!isPlainObject(c) || typeof c.cha_name !== 'string' || !c.cha_name) return []; return [{ - name: info.cha_name, - id: info.cid, - view_count: info.view_count, + name: c.cha_name, + id: c.cid ?? '', + view_count: c.view_count ?? 0, }]; }); if (list.length > 0 && rows.length === 0) { - throw new CommandExecutionError('douyin hashtag search: API returned challenges but none had stable challenge_info shape'); + throw new CommandExecutionError('douyin hashtag search: API returned suggestions but none had a stable shape'); } - return rows; + return rows.slice(0, kwargs.limit); } if (action === 'suggest') { const cover = String(kwargs.cover ?? '').trim(); diff --git a/clis/douyin/hashtag.test.js b/clis/douyin/hashtag.test.js index 04ffd350f..260140720 100644 --- a/clis/douyin/hashtag.test.js +++ b/clis/douyin/hashtag.test.js @@ -75,18 +75,67 @@ describe('douyin hashtag', () => { expect(url).not.toContain('keyword='); }); - it('search threads --keyword + count into the challenge/search URL', async () => { + it('search threads --keyword into the challenge suggestion URL', async () => { const registry = getRegistry(); const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag'); browserFetchMock.mockResolvedValueOnce({ - challenge_list: [{ challenge_info: { cha_name: '美食', cid: '123', view_count: 5000 } }], + sug_list: [{ cha_name: '美食', cid: '123', view_count: 5000 }], }); const rows = await cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 10 }); expect(rows).toEqual([{ name: '美食', id: '123', view_count: 5000 }]); const url = browserFetchMock.mock.calls[0][2]; - expect(url).toContain('challenge/search'); + expect(url).toContain('search/challengesug'); expect(url).toContain('keyword=' + encodeURIComponent('美食')); - expect(url).toContain('count=10'); + expect(url).not.toContain('challenge/search/'); + }); + + it('search keeps the composer parameters the endpoint is gated on', async () => { + const registry = getRegistry(); + const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag'); + browserFetchMock.mockResolvedValueOnce({ sug_list: [] }); + await cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 10 }); + const url = browserFetchMock.mock.calls[0][2]; + expect(url).toContain('source=challenge_create'); + expect(url).toContain('aid=2906'); + }); + + it('search typed-fails when the suggestion fields are renamed', async () => { + const registry = getRegistry(); + const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag'); + browserFetchMock.mockResolvedValueOnce({ + sug_list: [{ challenge_name: '美食', challenge_id: '1', view_cnt: 9 }], + }); + await expect(cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 10 })) + .rejects.toBeInstanceOf(CommandExecutionError); + }); + + it('search rejects a non-positive --limit before fetching', async () => { + const registry = getRegistry(); + const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag'); + await expect(cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 0 })) + .rejects.toBeInstanceOf(ArgumentError); + expect(browserFetchMock).not.toHaveBeenCalled(); + }); + + it('search fills --limit from the parsable entries, not the raw list head', async () => { + const registry = getRegistry(); + const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag'); + browserFetchMock.mockResolvedValueOnce({ + sug_list: [{ sug_type: 2 }, { cha_name: '美食', cid: '123', view_count: 5000 }], + }); + const rows = await cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 1 }); + expect(rows).toEqual([{ name: '美食', id: '123', view_count: 5000 }]); + }); + + it('search caps the fixed-size suggestion list at --limit', async () => { + const registry = getRegistry(); + const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag'); + browserFetchMock.mockResolvedValueOnce({ + sug_list: Array.from({ length: 11 }, (_, i) => ({ cha_name: 'tag' + i, cid: String(i), view_count: i })), + }); + const rows = await cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 3 }); + expect(rows).toHaveLength(3); + expect(rows[0]).toEqual({ name: 'tag0', id: '0', view_count: 0 }); }); it('suggest threads --cover into the hashtag/rec URL on success', async () => { @@ -113,16 +162,16 @@ describe('douyin hashtag', () => { it('search throws CommandExecutionError when challenge_list has wrong shape', async () => { const registry = getRegistry(); const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag'); - browserFetchMock.mockResolvedValueOnce({ challenge_list: 'not-an-array' }); + browserFetchMock.mockResolvedValueOnce({ sug_list: 'not-an-array' }); await expect(cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 10 })) .rejects.toBeInstanceOf(CommandExecutionError); }); - it('search throws CommandExecutionError when challenges return but none parse', async () => { + it('search throws CommandExecutionError when suggestions return but none parse', async () => { const registry = getRegistry(); const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag'); browserFetchMock.mockResolvedValueOnce({ - challenge_list: [{ challenge_info: null }, { other_field: 1 }], + sug_list: [null, 'not-an-object'], }); await expect(cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 10 })) .rejects.toBeInstanceOf(CommandExecutionError);