Skip to content

fix(douyin): search hashtags through the live endpoint and report empty responses #2254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Benjamin-eecs wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(douyin): search hashtags through the live endpoint and report empty responses #2254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion clis/douyin/_shared/browser-fetch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ export async function browserFetch(page, method, url, options = {}) {
${options.body ? `body: JSON.stringify(${JSON.stringify(options.body)}),` : ''}
});
const text = await res.text();
// A retired or gated endpoint answers 200 with no body. Reporting that
// as a parse failure sends readers after the JSON instead of the
// endpoint (issue #1405 fixed it, #1587 dropped it again).
if (!text.trim()) return res.ok ? null : { status_code: res.status, status_msg: 'Empty response body' };
try {
return JSON.parse(text);
} catch (error) {
Expand All @@ -47,7 +51,10 @@ export async function browserFetch(page, method, url, options = {}) {
throw new CommandExecutionError(`Douyin API request failed (${method} ${url}): ${error instanceof Error ? error.message : String(error)}`);
}
if (result == null) {
throw new CommandExecutionError(`Empty response from Douyin API (${method} ${url})`);
throw new CommandExecutionError(
`Empty response from Douyin API (${method} ${url})`,
'The endpoint may have been retired or may now require signed parameters.',
);
}
if (Array.isArray(result) || typeof result !== 'object') {
throw new CommandExecutionError(`Malformed response from Douyin API (${method} ${url})`);
Expand Down
44 changes: 44 additions & 0 deletions clis/douyin/_shared/browser-fetch.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ function makePage(result) {
screenshot: vi.fn(),
};
}
function makeScriptPage(body, { ok = true, status = 200, envelope = false } = {}) {
const page = makePage(null);
page.evaluate = vi.fn(async (script) => {
const stubFetch = async () => ({ ok, status, text: async () => body });
const value = await new Function('fetch', `return (${script});`)(stubFetch);
return envelope ? { session: 'site:douyin:test', data: value } : value;
});
return page;
}
describe('browserFetch', () => {
it('returns parsed JSON on success', async () => {
const page = makePage({ status_code: 0, data: { ak: 'KEY' } });
Expand All @@ -38,6 +47,41 @@ describe('browserFetch', () => {
const result = await browserFetch(page, 'GET', 'https://creator.douyin.com/api/test');
expect(result).toEqual({ some_field: 'value' });
});
it('reports an empty body as an empty response, not a parse failure', async () => {
const page = makeScriptPage('');
await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test'))
.rejects.toThrow('Empty response from Douyin API');
});
it('treats a whitespace-only body the same way', async () => {
const page = makeScriptPage(' \n ');
await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test'))
.rejects.toThrow('Empty response from Douyin API');
});
it('reports an empty body through a Browser Bridge {session,data} envelope', async () => {
const page = makeScriptPage('', { envelope: true });
await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test'))
.rejects.toThrow('Empty response from Douyin API');
});
it('still reports a non-JSON body as a parse failure', async () => {
const page = makeScriptPage('<html>gateway</html>');
await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test'))
.rejects.toThrow('JSON parse failed: <html>gateway</html>');
});
it('parses a JSON body from the generated script', async () => {
const page = makeScriptPage('{"status_code":0,"challenge_list":[]}');
await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test'))
.resolves.toEqual({ status_code: 0, challenge_list: [] });
});
it('keeps the auth classification when an empty body comes with 403', async () => {
const page = makeScriptPage('', { ok: false, status: 403 });
await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test'))
.rejects.toBeInstanceOf(AuthRequiredError);
});
it('keeps the status when an empty body comes with 404', async () => {
const page = makeScriptPage('', { ok: false, status: 404 });
await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test'))
.rejects.toThrow('Douyin API error 404');
});
it('throws on empty response body (null from evaluate)', async () => {
const page = makePage(null);
await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')).rejects.toThrow('Empty response from Douyin API');
Expand Down
24 changes: 15 additions & 9 deletions clis/douyin/hashtag.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ function requireListField(res, field, action) {

function validateHashtagArgs(kwargs) {
const action = kwargs.action;
const limit = Number(kwargs.limit ?? 10);
if (!Number.isInteger(limit) || limit < 1) {
throw new ArgumentError(`--limit must be a positive integer, got ${JSON.stringify(kwargs.limit)}`);
}
if (action === 'search') {
const keyword = String(kwargs.keyword ?? '').trim();
if (!keyword) {
Expand Down Expand Up @@ -55,22 +59,24 @@ cli({
const action = kwargs.action;
if (action === 'search') {
const keyword = String(kwargs.keyword ?? '').trim();
const url = `https://creator.douyin.com/aweme/v1/challenge/search/?keyword=${encodeURIComponent(keyword)}&count=${kwargs.limit}&aid=1128`;
// challenge/search answers 200 with an empty body; the creator
// studio composer reads suggestions from this endpoint instead,
// which ignores count and returns a fixed-size list (#2205).
const url = `https://creator.douyin.com/aweme/v1/search/challengesug/?keyword=${encodeURIComponent(keyword)}&source=challenge_create&aid=2906`;
const res = await browserFetch(page, 'GET', url);
const list = requireListField(res, 'challenge_list', 'search');
const list = requireListField(res, 'sug_list', 'search');
const rows = list.flatMap(c => {
const info = c?.challenge_info;
if (!isPlainObject(info)) return [];
if (!isPlainObject(c) || typeof c.cha_name !== 'string' || !c.cha_name) return [];
return [{
name: info.cha_name,
id: info.cid,
view_count: info.view_count,
name: c.cha_name,
id: c.cid ?? '',
view_count: c.view_count ?? 0,
}];
});
if (list.length > 0 && rows.length === 0) {
throw new CommandExecutionError('douyin hashtag search: API returned challenges but none had stable challenge_info shape');
throw new CommandExecutionError('douyin hashtag search: API returned suggestions but none had a stable shape');
}
return rows;
return rows.slice(0, kwargs.limit);
}
if (action === 'suggest') {
const cover = String(kwargs.cover ?? '').trim();
Expand Down
63 changes: 56 additions & 7 deletions clis/douyin/hashtag.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,18 +75,67 @@ describe('douyin hashtag', () => {
expect(url).not.toContain('keyword=');
});

it('search threads --keyword + count into the challenge/search URL', async () => {
it('search threads --keyword into the challenge suggestion URL', async () => {
const registry = getRegistry();
const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag');
browserFetchMock.mockResolvedValueOnce({
challenge_list: [{ challenge_info: { cha_name: '美食', cid: '123', view_count: 5000 } }],
sug_list: [{ cha_name: '美食', cid: '123', view_count: 5000 }],
});
const rows = await cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 10 });
expect(rows).toEqual([{ name: '美食', id: '123', view_count: 5000 }]);
const url = browserFetchMock.mock.calls[0][2];
expect(url).toContain('challenge/search');
expect(url).toContain('search/challengesug');
expect(url).toContain('keyword=' + encodeURIComponent('美食'));
expect(url).toContain('count=10');
expect(url).not.toContain('challenge/search/');
});

it('search keeps the composer parameters the endpoint is gated on', async () => {
const registry = getRegistry();
const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag');
browserFetchMock.mockResolvedValueOnce({ sug_list: [] });
await cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 10 });
const url = browserFetchMock.mock.calls[0][2];
expect(url).toContain('source=challenge_create');
expect(url).toContain('aid=2906');
});

it('search typed-fails when the suggestion fields are renamed', async () => {
const registry = getRegistry();
const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag');
browserFetchMock.mockResolvedValueOnce({
sug_list: [{ challenge_name: '美食', challenge_id: '1', view_cnt: 9 }],
});
await expect(cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 10 }))
.rejects.toBeInstanceOf(CommandExecutionError);
});

it('search rejects a non-positive --limit before fetching', async () => {
const registry = getRegistry();
const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag');
await expect(cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 0 }))
.rejects.toBeInstanceOf(ArgumentError);
expect(browserFetchMock).not.toHaveBeenCalled();
});

it('search fills --limit from the parsable entries, not the raw list head', async () => {
const registry = getRegistry();
const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag');
browserFetchMock.mockResolvedValueOnce({
sug_list: [{ sug_type: 2 }, { cha_name: '美食', cid: '123', view_count: 5000 }],
});
const rows = await cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 1 });
expect(rows).toEqual([{ name: '美食', id: '123', view_count: 5000 }]);
});

it('search caps the fixed-size suggestion list at --limit', async () => {
const registry = getRegistry();
const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag');
browserFetchMock.mockResolvedValueOnce({
sug_list: Array.from({ length: 11 }, (_, i) => ({ cha_name: 'tag' + i, cid: String(i), view_count: i })),
});
const rows = await cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 3 });
expect(rows).toHaveLength(3);
expect(rows[0]).toEqual({ name: 'tag0', id: '0', view_count: 0 });
});

it('suggest threads --cover into the hashtag/rec URL on success', async () => {
Expand All @@ -113,16 +162,16 @@ describe('douyin hashtag', () => {
it('search throws CommandExecutionError when challenge_list has wrong shape', async () => {
const registry = getRegistry();
const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag');
browserFetchMock.mockResolvedValueOnce({ challenge_list: 'not-an-array' });
browserFetchMock.mockResolvedValueOnce({ sug_list: 'not-an-array' });
await expect(cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 10 }))
.rejects.toBeInstanceOf(CommandExecutionError);
});

it('search throws CommandExecutionError when challenges return but none parse', async () => {
it('search throws CommandExecutionError when suggestions return but none parse', async () => {
const registry = getRegistry();
const cmd = [...registry.values()].find((c) => c.site === 'douyin' && c.name === 'hashtag');
browserFetchMock.mockResolvedValueOnce({
challenge_list: [{ challenge_info: null }, { other_field: 1 }],
sug_list: [null, 'not-an-object'],
});
await expect(cmd.func({}, { action: 'search', keyword: '美食', cover: '', limit: 10 }))
.rejects.toBeInstanceOf(CommandExecutionError);
Expand Down