From a808143c04cca8706d53ce2724bdd063b5334b6e Mon Sep 17 00:00:00 2001 From: xin-yi33 Date: Wed, 24 Jun 2026 11:04:54 +0800 Subject: [PATCH] feat: add GUI workflow scripts alongside existing MCP Server (CLI) - Add cdp-client.mjs: shared CDP connection layer (GUI and CLI) - Add discover-apis.mjs: discover API endpoints via browser - Add intercept-response.mjs: intercept and modify API responses - Add bulk-collect.mjs: paginated data collection with resume support - Update README with GUI scripts usage examples Co-Authored-By: Claude Opus 4.7 --- README.md | 24 +++++++ scripts/gui/TODO.md | 31 +++++++++ scripts/gui/bulk-collect.mjs | 75 +++++++++++++++++++++ scripts/gui/cdp-client.mjs | 101 +++++++++++++++++++++++++++++ scripts/gui/discover-apis.mjs | 70 ++++++++++++++++++++ scripts/gui/intercept-response.mjs | 81 +++++++++++++++++++++++ 6 files changed, 382 insertions(+) create mode 100644 scripts/gui/TODO.md create mode 100644 scripts/gui/bulk-collect.mjs create mode 100644 scripts/gui/cdp-client.mjs create mode 100644 scripts/gui/discover-apis.mjs create mode 100644 scripts/gui/intercept-response.mjs diff --git a/README.md b/README.md index 2649f3b..e43f9cd 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,30 @@ npm run build | `evaluate_script` | 在页面或断点上下文执行 JavaScript，支持主世界、保存结果和读取一个本地输入文件 | | `list_console_messages` | 列出控制台消息，或按 msgid 获取单条详情 | +## GUI 工作流脚本 + +除了 MCP Server 模式（CLI），项目还提供独立的 GUI 自动化脚本，用于浏览器操作、API 发现和数据采集。**GUI 和 CLI 共享同一套 CDP 连接层，可以同时使用。** + +| 脚本 | 用途 | +|------|------| +| `scripts/gui/cdp-client.mjs` | CDP 连接层（公共模块，GUI/CLI 共享） | +| `scripts/gui/discover-apis.mjs` | API 端点发现 | +| `scripts/gui/intercept-response.mjs` | 网络响应拦截与篡改 | +| `scripts/gui/bulk-collect.mjs` | 批量分页数据采集 | + +```bash +# API 发现 +node scripts/gui/discover-apis.mjs --url https://example.com --port 9222 + +# 响应拦截（绕过地域/身份检测） +node scripts/gui/intercept-response.mjs --url https://example.com --pattern *api* --transform 'data.userProvince="湖北"' + +# 批量采集 +node scripts/gui/bulk-collect.mjs --url "https://api.example.com/list?page=" --port 9222 --pages 10 --output data.json +``` + +详见 [scripts/gui/TODO.md](scripts/gui/TODO.md)。 + ## 使用示例 ### JS 逆向基本流程 diff --git a/scripts/gui/TODO.md b/scripts/gui/TODO.md new file mode 100644 index 0000000..50ad12f --- /dev/null +++ b/scripts/gui/TODO.md @@ -0,0 +1,31 @@ +# GUI 工作流脚本 — 后续规划 + +已完成的基础模块： + +- [x] cdp-client.mjs — CDP 连接层（GUI/CLI 共享） +- [x] discover-apis.mjs — API 端点发现 +- [x] intercept-response.mjs — 网络响应拦截与篡改 +- [x] bulk-collect.mjs — 批量分页采集 + +待完善： + +- [ ] page-navigate.mjs — 多步骤导航链（支持条件等待、重试） +- [ ] capture-screenshot.mjs — 自动截图（可调用 MCP Server 的 take_screenshot 复用） +- [ ] extract-table.mjs — 从页面提取表格数据 +- [ ] evaluate-flow.mjs — 执行 JS 并跟踪返回值变化 +- [ ] export-to-curl.mjs — 将捕获的请求导出为 curl 命令 + +与 MCP Server 整合： + +- [ ] 在 package.json 中添加 `scripts/gui/` 路径 +- [ ] 统一 CLI 参数格式（与 MCP CLI 的 flag 风格对齐） +- [ ] 新增 `--gui` 启动参数，启动后进入交互式 GUI 模式提示 + +## 设计理念 + +GUI 脚本遵循以下原则： + +1. **不重复实现 MCP Server 已有功能** — 断点、源码分析、Console 使用 MCP Server 的 21 个工具 +2. **提供更高层的工作流封装** — API 发现、批量采集、响应拦截是跨场景的通用步骤 +3. **与 CLI 模式共享 CDP 连接** — 所有脚本都基于 `cdp-client.mjs`，该模块也可被 MCP Server 内部引用 +4. **浏览器实例复用** — GUI 脚本和 MCP Server 可以连接同一个浏览器，互不干扰 diff --git a/scripts/gui/bulk-collect.mjs b/scripts/gui/bulk-collect.mjs new file mode 100644 index 0000000..1a1394e --- /dev/null +++ b/scripts/gui/bulk-collect.mjs @@ -0,0 +1,75 @@ +#!/usr/bin/env node +/** + * 批量数据采集工具 - GUI 模式 + * + * 从分页 API 批量采集数据，支持自动翻页和断点续采。 + * + * 用法: + * node scripts/gui/bulk-collect.mjs --url "https://api.example.com/list?page=" --port 9222 --pages 10 + * node scripts/gui/bulk-collect.mjs --url "https://api.example.com/list" --port 9222 --pages all + */ + +import { CDPClient, findPageWs } from './cdp-client.mjs'; +import fs from 'fs'; + +const args = process.argv.slice(2); +const targetUrl = args.includes('--url') ? args[args.indexOf('--url') + 1] : ''; +const cdpPort = args.includes('--port') ? args[args.indexOf('--port') + 1] : '9222'; +const pageParam = args.includes('--pages') ? args[args.indexOf('--pages') + 1] : '5'; +const outputFile = args.includes('--output') ? args[args.indexOf('--output') + 1] : 'collected_data.json'; +const delay = parseInt(args.includes('--delay') ? args[args.indexOf('--delay') + 1] : '500'); + +async function main() { + if (!targetUrl) { console.error('Usage: --url '); process.exit(1); } + + const wsUrl = await findPageWs(`http://127.0.0.1:${cdpPort}`, targetUrl); + if (!wsUrl) { console.error('No page found'); process.exit(1); } + const cdp = new CDPClient(wsUrl); + await cdp.send('Network.enable'); + + // 收集响应 + const responses = []; + const pendingResponses = new Map(); + + cdp._ws.on('message', (data) => { + try { + const msg = JSON.parse(data.toString()); + if (msg.method === 'Network.responseReceived') { + const url = msg.params?.response?.url || ''; + if (url.includes(targetUrl.split('?')[0])) { + const reqId = msg.params.requestId; + setTimeout(async () => { + try { + const body = await cdp.send('Network.getResponseBody', { requestId: reqId }); + if (body?.result?.body) { + try { + responses.push(JSON.parse(body.result.body)); + } catch(e) { + responses.push({ raw: body.result.body.substring(0, 500) }); + } + } + } catch(e) {} + }, delay); + } + } + } catch(e) {} + }); + + // 计算页数 + const pages = pageParam === 'all' ? 100 : parseInt(pageParam); + const hasPagePlaceholder = targetUrl.includes('page='); + + for (let p = 1; p <= pages; p++) { + const url = hasPagePlaceholder ? targetUrl.replace(/page=\d+/, `page=${p}`) : `${targetUrl}${p}`; + console.log(`Page ${p}/${pages}...`); + await cdp.send('Page.navigate', { url }); + await CDPClient.sleep(delay * 2); + } + + // 保存结果 + fs.writeFileSync(outputFile, JSON.stringify(responses, null, 2)); + console.log(`\nDone! ${responses.length} responses saved to ${outputFile}`); + process.exit(0); +} + +main().catch(e => { console.error('Error:', e); process.exit(1); }); diff --git a/scripts/gui/cdp-client.mjs b/scripts/gui/cdp-client.mjs new file mode 100644 index 0000000..2263aa6 --- /dev/null +++ b/scripts/gui/cdp-client.mjs @@ -0,0 +1,101 @@ +/** + * CDP Client - 浏览器 DevTools Protocol 连接层 + * + * GUI/CLI 共享模块。MCP Server 内部和 GUI 脚本均使用此模块连接和操作浏览器。 + * + * 用法: + * import { CDPClient } from './cdp-client.mjs'; + * const client = new CDPClient('ws://127.0.0.1:9222/...'); + * await client.send('Page.navigate', { url: 'https://example.com' }); + * const title = await client.eval('document.title'); + */ + +import WebSocket from 'ws'; + +export class CDPClient { + constructor(wsUrl) { + this._id = 1; + this._pending = new Map(); + this._ws = new WebSocket(wsUrl); + return new Promise((resolve, reject) => { + this._ws.on('open', () => resolve(this)); + this._ws.on('error', reject); + this._ws.on('message', (data) => { + try { + const msg = JSON.parse(data.toString()); + const resolve = this._pending.get(msg.id); + if (resolve) { this._pending.delete(msg.id); resolve(msg); } + } catch(e) {} + }); + }); + } + + /** 调用 CDP 方法 */ + async send(method, params = {}) { + return new Promise((resolve, reject) => { + const id = this._id++; + const timer = setTimeout(() => { this._pending.delete(id); reject(new Error(`Timeout: ${method}`)); }, 30000); + this._pending.set(id, (msg) => { clearTimeout(timer); resolve(msg); }); + this._ws.send(JSON.stringify({ id, method, params })); + }); + } + + /** 在页面执行 JS */ + async eval(expression) { + const r = await this.send('Runtime.evaluate', { expression, returnByValue: true }); + return r?.result?.result?.value; + } + + /** 模拟鼠标点击 */ + async clickAt(x, y) { + await this.send('Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 }); + await this.send('Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 }); + } + + /** 按 CSS 选择器点击 */ + async clickEl(selector) { + const coords = await this.eval(`JSON.stringify((()=>{ + const el = document.querySelector('${selector}'); + if (!el) return null; + const rect = el.getBoundingClientRect(); + return { x: Math.round(rect.x + rect.width / 2), y: Math.round(rect.y + rect.height / 2) }; + })())`); + if (!coords) throw new Error(`Element not found: ${selector}`); + const { x, y } = JSON.parse(coords); + await this.clickAt(x, y); + } + + /** 按文本内容点击 */ + async clickByText(text) { + const coords = await this.eval(`JSON.stringify((()=>{ + const all = document.querySelectorAll('*'); + for (const el of all) { + if (el.textContent.trim() === '${text}' && el.offsetParent !== null) { + const rect = el.getBoundingClientRect(); + return { x: Math.round(rect.x + rect.width / 2), y: Math.round(rect.y + rect.height / 2) }; + } + } + return null; + })())`); + if (!coords) throw new Error(`Text not found: ${text}`); + const { x, y } = JSON.parse(coords); + await this.clickAt(x, y); + } + + /** 等待指定毫秒 */ + static sleep(ms) { return new Promise(r => setTimeout(r, ms)); } +} + +/** 从 CDP HTTP 端点获取页面 WebSocket URL */ +export async function findPageWs(httpEndpoint, urlFilter) { + const resp = await fetch(`${httpEndpoint}/json`); + const pages = await resp.json(); + const target = pages.find(p => p.url && (!urlFilter || p.url.includes(urlFilter))); + return target?.webSocketDebuggerUrl; +} + +/** 从 CDP HTTP 端点获取所有页面 */ +export async function listPages(httpEndpoint) { + const resp = await fetch(`${httpEndpoint}/json`); + return resp.json(); +} diff --git a/scripts/gui/discover-apis.mjs b/scripts/gui/discover-apis.mjs new file mode 100644 index 0000000..37c1711 --- /dev/null +++ b/scripts/gui/discover-apis.mjs @@ -0,0 +1,70 @@ +#!/usr/bin/env node +/** + * API 发现工具 - GUI 模式 + * + * 连接到浏览器，导航到目标页面，自动捕获所有网络请求中的 API 端点。 + * + * 用法: + * node scripts/gui/discover-apis.mjs --url https://example.com --port 9222 + * node scripts/gui/discover-apis.mjs --url https://example.com --filter api/ + */ + +import { CDPClient, findPageWs } from './cdp-client.mjs'; + +const args = process.argv.slice(2); +const targetUrl = args.includes('--url') ? args[args.indexOf('--url') + 1] : 'https://example.com'; +const cdpPort = args.includes('--port') ? args[args.indexOf('--port') + 1] : '9222'; +const urlFilter = args.includes('--filter') ? args[args.indexOf('--filter') + 1] : 'api/'; + +async function main() { + // 1. 连接浏览器 + const wsUrl = await findPageWs(`http://127.0.0.1:${cdpPort}`, targetUrl); + if (!wsUrl) { + console.error(`No page found on port ${cdpPort}`); + process.exit(1); + } + const cdp = new CDPClient(wsUrl); + + // 2. 启用网络捕获 + await cdp.send('Network.enable'); + + // 3. 收集 API 请求 + const apis = new Map(); + cdp._ws.on('message', (data) => { + try { + const msg = JSON.parse(data.toString()); + if (msg.method === 'Network.requestWillBeSent') { + const url = msg.params?.request?.url || ''; + if (url.includes(urlFilter)) { + const key = url.split('?')[0]; + if (!apis.has(key)) { + apis.set(key, { + method: msg.params.request.method, + url: key, + headers: msg.params.request.headers, + postData: msg.params.request.postData, + }); + } + } + } + } catch(e) {} + }); + + // 4. 导航到目标页面 + console.log(`Navigating to ${targetUrl}...`); + await cdp.send('Page.navigate', { url: targetUrl }); + await CDPClient.sleep(5000); + + // 5. 输出结果 + console.log(`\nFound ${apis.size} API endpoints:`); + for (const [url, info] of apis) { + console.log(` ${info.method} ${url}`); + if (info.postData) { + console.log(` POST body: ${info.postData.substring(0, 200)}`); + } + } + + process.exit(0); +} + +main().catch(e => { console.error('Error:', e); process.exit(1); }); diff --git a/scripts/gui/intercept-response.mjs b/scripts/gui/intercept-response.mjs new file mode 100644 index 0000000..9a2334e --- /dev/null +++ b/scripts/gui/intercept-response.mjs @@ -0,0 +1,81 @@ +#!/usr/bin/env node +/** + * 网络响应拦截工具 - GUI 模式 + * + * 拦截页面 API 请求的响应，实时修改后返回给页面。 + * 可用于绕过服务端地域/身份检测，或模拟不同响应场景。 + * + * 用法: + * node scripts/gui/intercept-response.mjs --url https://example.com --port 9222 --pattern *target-api* --transform 'data.userProvince="湖北"' + */ + +import { CDPClient, findPageWs } from './cdp-client.mjs'; + +const args = process.argv.slice(2); +const targetUrl = args.includes('--url') ? args[args.indexOf('--url') + 1] : 'https://example.com'; +const cdpPort = args.includes('--port') ? args[args.indexOf('--port') + 1] : '9222'; +const urlPattern = args.includes('--pattern') ? args[args.indexOf('--pattern') + 1] : '*'; +const transformExpr = args.includes('--transform') ? args[args.indexOf('--transform') + 1] : ''; + +async function main() { + const wsUrl = await findPageWs(`http://127.0.0.1:${cdpPort}`, targetUrl); + if (!wsUrl) { console.error(`No page found`); process.exit(1); } + const cdp = new CDPClient(wsUrl); + + // 启用请求拦截 + await cdp.send('Fetch.enable', { + patterns: [{ urlPattern, requestStage: 'Response' }] + }); + + let interceptedCount = 0; + + cdp._ws.on('message', async (data) => { + try { + const msg = JSON.parse(data.toString()); + if (msg.method === 'Fetch.requestPaused') { + const reqId = msg.params.requestId; + const url = msg.params.request?.url || ''; + + if (transformExpr) { + // 获取原始响应体 + const bodyResp = await cdp.send('Fetch.getResponseBody', { requestId: reqId }); + let responseBody = bodyResp?.result?.body || '{}'; + + try { + const parsed = JSON.parse(responseBody); + // 应用变换（简化版：通过 Function 执行用户定义的变换表达式） + const transform = new Function('data', `try { ${transformExpr}; return data; } catch(e) { return data; }`); + const modified = transform(parsed); + responseBody = JSON.stringify(modified); + interceptedCount++; + console.log(` Intercepted: ${url.substring(0, 80)}`); + } catch(e) {} + + await cdp.send('Fetch.fulfillRequest', { + requestId: reqId, + responseCode: 200, + responseHeaders: [{ name: 'Content-Type', value: 'application/json; charset=utf-8' }], + body: Buffer.from(responseBody).toString('base64') + }); + } else { + await cdp.send('Fetch.continueRequest', { requestId: reqId }); + } + } + } catch(e) {} + }); + + // 导航到页面 + console.log(`Navigating to ${targetUrl}...`); + await cdp.send('Page.navigate', { url: targetUrl }); + await CDPClient.sleep(10000); + + console.log(`\nIntercepted ${interceptedCount} requests`); + if (!transformExpr) { + console.log('Tip: Use --transform to modify responses'); + console.log(' node scripts/gui/intercept-response.mjs --url URL --pattern *api* --transform \'data.userProvince="湖北"\''); + } + + process.exit(0); +} + +main().catch(e => { console.error('Error:', e); process.exit(1); });