diff --git a/CHANGELOG.md b/CHANGELOG.md index 30f6ed7..8d7e453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Fixed +- Fix from `main` PR #112: consult answers came back with `model_used` and `pro_verified` missing, so a Pro consult could not be told apart from one nobody checked. ChatGPT stopped tagging answers with the model that wrote them: measured 2026-11-17 on this container's tab, the page had no `data-message-model-slug` anywhere, and neither the answer's buttons nor its More actions menu name a model. prodex reads rendered page content only, so the answering model cannot be read. `model_used` is now `"unknown"` in that case, a `model_unverified` warning says why `pro_verified` is left out, and the new `picker_effort` field (CLI JSON, MCP `pro_consult`, the receipt and a `picker_effort:` line) records the step the picker's effort slider read once prodex had set it, before the prompt was typed. That shows what was selected, not what answered. - Fix from `main` PR #109: the MCP bridge tools refused Korean text at about a third of the limit they advertise. `tools/list` gives each text field a `maxLength` in characters (10,000 for titles, ids and paths, 100,000 for prompts, summaries, commands and warnings), but the tools counted UTF-8 bytes, and a Korean character is three. Found 2026-11-09: a `bridge_create_task` title of 4,000 "가" was refused as "MCP title is too large (12000 bytes > 10000 bytes)", and a prompt of 40,000 as "(120000 bytes > 100000 bytes)". They now count characters, as the schemas do. `repo_write_file_dry_run` content stays limited to 1,000,000 bytes, the size of file the repo tools read and write: its schema now says so, and the refusal gives the size in bytes. - Fix from `main` PR #109: `bridge_create_task` with `repo_id: ""` came back as the bridge store's raw validation dump (`[ { "code": "too_small", ... "path": [ "repo_id" ] } ]`). Found 2026-11-09: the tool schema allowed the empty string and the store requires at least one character. The schema now advertises `minLength: 1` for `repo_id`, so an empty one is refused as invalid arguments, as an empty title is. - Fix from `main` PR #109: `prodex setup` printed a follow-up command that did not work in two cases. Found 2026-11-09: with a saved token that had expired, setup printed "Token expired: ...", then "it never expires" and `prodex status --show-token --url-only --unsafe-show-non-expiring-token`, which refuses an expired token; `--token-ttl-hours 1e-9` made such a token and exited 0. And `prodex setup --cwd ` run from another directory printed the status command without `--cwd`, so running it as printed failed with "status requires local MCP setup". An expired token now gets `prodex setup --token-ttl-hours `, the commands setup prints carry the `--cwd` it was given, and a TTL too small to outlast the moment the token is made is refused, as a TTL of 0 is. diff --git a/docs/clients.md b/docs/clients.md index 9b994bb..53ea236 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -79,8 +79,12 @@ set this flag, carry it into later calls, or switch chats/keys to evade the chec Approval is caller attestation, not independent human authentication. Manual CLI calls and other bridge roots are outside this cooperative MCP guard. -The response exposes `model_used`, `pro_verified`, `continued_from`, `request_id`, -and `request_verified` when available. A ready-to-use `continuation` is only offered +The response exposes `model_used`, `pro_verified`, `picker_effort`, `continued_from`, +`request_id`, and `request_verified` when available. Since November 2026 the ChatGPT +page no longer names the model that wrote an answer, so `model_used` is usually +`"unknown"`, `pro_verified` is left out, and a `model_unverified` warning says so. +`picker_effort` is the step the picker's effort slider read before the prompt was +typed: evidence of what was selected, not of what answered. A ready-to-use `continuation` is only offered for a saved, request-verified conversation answer (or an approval checkpoint's already-resolved target). An answer that failed to save or is incomplete does not invite another automatic turn; report it and resolve the blocker first. diff --git a/src/chatgpt-browser.ts b/src/chatgpt-browser.ts index cb59aef..fbd710e 100644 --- a/src/chatgpt-browser.ts +++ b/src/chatgpt-browser.ts @@ -347,6 +347,11 @@ export interface SendChatGptPromptResult { modelHints: string[]; /** The model ChatGPT says produced the answer, e.g. "gpt-5-6-pro". */ modelSlug?: string; + /** + * The step the picker's effort slider read once prodex had moved it, before + * the prompt was typed. Evidence of what was selected, not of what answered. + */ + pickerEffort?: string; /** * The project id the composer was confirmed bound to before typing, so the * caller can check the answered thread against where the prompt was aimed @@ -3605,7 +3610,8 @@ export function sliderWalkPlan(state: { position?: number; min?: number; max?: n async function selectModelReasoning( cdp: CdpConnection, options: Pick, - selectionWarnings: string[] = [] + selectionWarnings: string[] = [], + selectionEvidence: { effort?: string } = {} ): Promise { if (!options.model && !options.proMode && !options.effort) return; // --pro-mode selects a Pro sub-mode, so it is meaningless with a non-Pro @@ -3705,7 +3711,8 @@ async function selectModelReasoning( throw new Error("ChatGPT's model picker would not reopen after the model was chosen."); } try { - await selectPowerStepWithGrace(cdp, plan.sliderLabel); + const reached = await selectPowerStepWithGrace(cdp, plan.sliderLabel); + if (reached.effort) selectionEvidence.effort = reached.effort; } catch (error) { // Only "this slider has no such step" is the picker declining. A // slider that will not open, or will not move - the hydrating case, @@ -4610,6 +4617,7 @@ export async function sendChatGptPrompt(options: SendChatGptPromptOptions): Prom let submitButtonFound = false; let submissionAttempted = false; const sendWarnings: string[] = [...preflightWarnings]; + const selectionEvidence: { effort?: string } = {}; // Anything the page put in front of prodex was answered on the caller's // behalf. The note is read at return time - there are several return paths - // so it is folded into the array rather than pushed from each of them. @@ -4666,7 +4674,7 @@ export async function sendChatGptPrompt(options: SendChatGptPromptOptions): Prom } boundProjectId = await selectProject(cdp, options); try { - await selectModelReasoning(cdp, options, sendWarnings); + await selectModelReasoning(cdp, options, sendWarnings, selectionEvidence); } catch (modelError) { // Pro sub-mode isn't exposed in this UI yet (staged rollout). Pro itself is // already selected by this point, so degrade to plain Pro with a warning @@ -4949,6 +4957,7 @@ export async function sendChatGptPrompt(options: SendChatGptPromptOptions): Prom title: completed.title, answer: CHATGPT_NON_TEXT_ANSWER_NOTE, modelHints: completed.modelHints, + ...(selectionEvidence.effort ? { pickerEffort: selectionEvidence.effort } : {}), ...(boundProjectId ? { boundProjectId } : {}), requestId, requestVerified: true, @@ -4963,6 +4972,7 @@ export async function sendChatGptPrompt(options: SendChatGptPromptOptions): Prom answer: completed.answer.trim(), modelHints: completed.modelHints, ...(completed.modelSlug ? { modelSlug: completed.modelSlug } : {}), + ...(selectionEvidence.effort ? { pickerEffort: selectionEvidence.effort } : {}), ...(boundProjectId ? { boundProjectId } : {}), requestId, requestVerified: true, @@ -4982,6 +4992,7 @@ export async function sendChatGptPrompt(options: SendChatGptPromptOptions): Prom answer: completed.answer.trim(), modelHints: completed.modelHints, ...(completed.modelSlug ? { modelSlug: completed.modelSlug } : {}), + ...(selectionEvidence.effort ? { pickerEffort: selectionEvidence.effort } : {}), ...(boundProjectId ? { boundProjectId } : {}), requestId, requestVerified: true, diff --git a/src/cli-pro.ts b/src/cli-pro.ts index ec3651a..7a3020d 100644 --- a/src/cli-pro.ts +++ b/src/cli-pro.ts @@ -1893,11 +1893,14 @@ export async function runAskProCommand( if (consult.requestVerified === false && !persistenceWarnings.some((warning) => warning.startsWith("request_unverified:"))) { persistenceWarnings.push("request_unverified: the saved answer was not verified against its requested user turn. Review the original conversation before continuing."); } + if (!consult.modelSlug) persistenceWarnings.push(modelUnverifiedWarning(consult.pickerEffort)); + const modelUsed = consult.modelSlug ?? "unknown"; // Truncation and other send warnings must be visible at runtime, not // only inside the persisted receipt: a caller who never opens .bridge // would otherwise treat a cut-off answer as complete. for (const warning of persistenceWarnings) io.stderr(warning); - if (consult.modelSlug) io.stderr(`model_used: ${consult.modelSlug}`); + io.stderr(`model_used: ${modelUsed}`); + if (consult.pickerEffort) io.stderr(`picker_effort: ${consult.pickerEffort}`); if (consult.requestId) io.stderr(`request_id: ${consult.requestId}`); if (consult.requestVerified !== undefined) io.stderr(`request_verified: ${consult.requestVerified ? "yes" : "no"}`); // Which conversation this followed, and where the answer actually landed. @@ -1944,7 +1947,8 @@ export async function runAskProCommand( ...(Object.keys(selectionMetadata).length > 0 ? { selection: selectionMetadata } : {}), // What actually answered, straight from ChatGPT's own tag - the // receipt used to record only what prodex asked for. - ...(consult.modelSlug ? { model_used: consult.modelSlug } : {}), + model_used: modelUsed, + ...(consult.pickerEffort ? { picker_effort: consult.pickerEffort } : {}), ...(proVerified !== undefined ? { pro_verified: proVerified } : {}), ...(consult.requestId ? { request_id: consult.requestId } : {}), ...(consult.requestVerified !== undefined ? { request_verified: consult.requestVerified } : {}), @@ -2012,7 +2016,8 @@ export async function runAskProCommand( ...(consult.requestId ? { request_id: consult.requestId } : {}), ...(consult.requestVerified !== undefined ? { request_verified: consult.requestVerified } : {}), ...(continuedFromTaskId ? { continued_from: continuedFromTaskId } : {}), - ...(consult.modelSlug ? { model_used: consult.modelSlug } : {}), + model_used: modelUsed, + ...(consult.pickerEffort ? { picker_effort: consult.pickerEffort } : {}), ...(proVerified !== undefined ? { pro_verified: proVerified } : {}), destination: { observed: destination.destination, @@ -2138,6 +2143,8 @@ export interface BrowserConsultOutcome { request_id?: string; request_verified?: boolean; model_used?: string; + /** The step the picker's effort slider read before the prompt was typed. */ + picker_effort?: string; pro_verified?: boolean; continued_from?: string; continuation?: Pick; @@ -2948,6 +2955,22 @@ export function proSelectionVerified(selection: { return /pro/i.test(selection.modelSlug); } +/** + * Why model_used is "unknown" and pro_verified is missing. ChatGPT stopped + * tagging answers with the model that wrote them (measured 2026-11-17: no + * data-message-model-slug anywhere on the page, and nothing else on screen + * names it), and prodex reads rendered page content only. Leaving both fields + * out without a word read as "not checked" rather than "cannot be checked". + */ +export function modelUnverifiedWarning(pickerEffort: string | undefined): string { + return ( + 'model_unverified: the ChatGPT page did not name the model that wrote this answer, so model_used is "unknown" and pro_verified is left out.' + + (pickerEffort + ? ` Before the prompt was typed, the picker's effort slider read "${pickerEffort}"; that shows what was selected, not what answered.` + : "") + ); +} + /** * A temporary chat is not saved; a project chat is. Entering a project leaves * temporary mode, so asking for both is asking for two different things, and diff --git a/src/mcp.ts b/src/mcp.ts index d22a4d9..ec5ebfa 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -394,7 +394,7 @@ export function createServer(cwd = process.cwd(), options: CreateMcpServerOption .boolean() .optional() .describe( - "Send even when the requested EFFORT step could not be applied. Off by default, because an answer from a step nobody asked for is usually unusable: a consult asking for Pro that comes back from a lesser model cannot be cited as a Pro review. Pass true only when any answer beats no answer. It does not gate the model axis: a model the picker does not offer is a warning and the send goes out on whatever the composer had - check `model_used` and `pro_verified` in the answer to see what actually replied." + "Send even when the requested EFFORT step could not be applied. Off by default, because an answer from a step nobody asked for is usually unusable: a consult asking for Pro that comes back from a lesser model cannot be cited as a Pro review. Pass true only when any answer beats no answer. It does not gate the model axis: a model the picker does not offer is a warning and the send goes out on whatever the composer had - check `model_used` and `pro_verified` in the answer to see what actually replied. When the page does not name the model, `model_used` is \"unknown\" and `picker_effort` shows the step the picker read before typing." ) } }, diff --git a/tests/chatgpt-browser.test.ts b/tests/chatgpt-browser.test.ts index 0d1baf2..dfad61b 100644 --- a/tests/chatgpt-browser.test.ts +++ b/tests/chatgpt-browser.test.ts @@ -435,6 +435,30 @@ Show more`; expect(triggerReads).toBe(0); }); + it("returns the step the power slider read once the walk reached it", async () => { + // The answer no longer names its model, so this reading, taken before the + // prompt is typed, is the only evidence of what was selected. + vi.useFakeTimers(); + const thread = "https://chatgpt.com/c/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + installFakeChatGptSendCdp(thread, [ + { ...fakeAnswerState(thread, "", false), assistantMessageCount: 0, userMessageCount: 0 }, + fakeAnswerState(thread, "correct answer", false) + ]); + const base = FakeCdpWebSocket.evaluate; + FakeCdpWebSocket.evaluate = (expression) => { + if (expression === modelButtonRectExpression()) return { ok: true, x: 20, y: 20 }; + if (expression === menuOpenExpression() || expression.includes("document.elementFromPoint")) return true; + if (expression === powerSliderPresentExpression()) return true; + if (expression === powerSliderStateExpression()) return { ok: true, position: 4, min: 0, max: 4, effort: "Pro" }; + if (expression === focusPowerSliderExpression()) return { ok: true }; + return base(expression); + }; + const send = sendChatGptPrompt({ port: 19338, prompt: "answer this", effort: "Pro", targetUrl: thread, timeoutMs: 10_000 }); + void send.catch(() => undefined); + await vi.advanceTimersByTimeAsync(30_000); + await expect(send).resolves.toMatchObject({ answer: "correct answer", pickerEffort: "Pro" }); + }); + it.each([ "https://chatgpt.com/c/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", "https://chatgpt.com/g/g-p-1234567890123456-project/c/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" diff --git a/tests/cli-pro-browser-send.test.ts b/tests/cli-pro-browser-send.test.ts index 1db4489..64460b4 100644 --- a/tests/cli-pro-browser-send.test.ts +++ b/tests/cli-pro-browser-send.test.ts @@ -1566,8 +1566,9 @@ describe("pro browser ask persistence", () => { task_id: taskId, thread: "https://chatgpt.com/c/abc", // The send passed no model flag and this repo has no saved default, so - // the model-selection nudge is recorded alongside the send warning. - warnings: ["model hint observed", expect.stringContaining("model_selection_warning")] + // the model-selection nudge is recorded alongside the send warning, and + // the answer named no model, which is said too. + warnings: ["model hint observed", expect.stringContaining("model_selection_warning"), expect.stringContaining("model_unverified")] }) ); const result = JSON.parse(await readFile(path.join(cwd, ".bridge", "results", `${taskId}.json`), "utf8")) as { @@ -3034,7 +3035,7 @@ describe("pro browser ask model/project selection", () => { expect(payload.thread).toBe("https://chatgpt.com/c/json"); expect(payload.answer).toBe("json answer"); expect(payload.session_key).toBe("codex-thread-json"); - expect(payload.warnings).toEqual([expect.stringContaining("model_selection_warning")]); + expect(payload.warnings).toEqual([expect.stringContaining("model_selection_warning"), expect.stringContaining("model_unverified")]); }); it("starts an ordinary consult in a fresh chat while keeping the default project", async () => { diff --git a/tests/mcp-consult.test.ts b/tests/mcp-consult.test.ts index d3cf9bd..f0fa809 100644 --- a/tests/mcp-consult.test.ts +++ b/tests/mcp-consult.test.ts @@ -413,6 +413,57 @@ describe("performBrowserConsultForMcp", () => { expect(outcome.notes.some((note) => note.startsWith("answer_incomplete:"))).toBe(true); }); + // Measured 2026-11-17: ChatGPT stopped tagging answers with the model that + // wrote them, and a Pro consult came back with model_used and pro_verified + // simply missing. The caller could not tell "not checked" from "checked and + // unknowable", so it is said, with the one piece of evidence prodex has. + it("says the answering model is unknown when the page does not name it, and shows what the picker read", async () => { + const cwd = await mkdtemp(path.join(tmpdir(), "prodex-mcp-consult-")); + sendChatGptPromptMock.mockResolvedValueOnce({ + url: "https://chatgpt.com/c/mcp-untagged", + title: "ChatGPT", + answer: "untagged answer", + modelHints: [], + pickerEffort: "Pro", + warnings: [], + requestId: "9cb9650622e74a62bd9074c42a311945", + requestVerified: true + }); + + const outcome = await performBrowserConsultForMcp(cwd, { prompt: "Untagged check", effort: "Pro" }); + const { BridgeStore } = await import("../src/store.js"); + const receipts = await new BridgeStore(cwd).listReceipts({ kind: "consult_answer_saved", task_id: outcome.task_id }); + + expect(outcome.model_used).toBe("unknown"); + expect(outcome.pro_verified).toBeUndefined(); + expect(outcome.picker_effort).toBe("Pro"); + const warning = outcome.warnings?.find((line) => line.startsWith("model_unverified: ")); + expect(warning).toContain('"Pro"'); + expect(receipts[0]?.metadata).toEqual(expect.objectContaining({ model_used: "unknown", picker_effort: "Pro" })); + expect(receipts[0]?.metadata).not.toHaveProperty("pro_verified"); + }); + + it("keeps quiet about the model when the page names it", async () => { + const cwd = await mkdtemp(path.join(tmpdir(), "prodex-mcp-consult-")); + sendChatGptPromptMock.mockResolvedValueOnce({ + url: "https://chatgpt.com/c/mcp-tagged", + title: "ChatGPT", + answer: "tagged answer", + modelHints: [], + modelSlug: "gpt-6-pro", + pickerEffort: "Pro", + warnings: [], + requestId: "9cb9650622e74a62bd9074c42a311945", + requestVerified: true + }); + + const outcome = await performBrowserConsultForMcp(cwd, { prompt: "Tagged check", effort: "Pro" }); + + expect(outcome.model_used).toBe("gpt-6-pro"); + expect(outcome.pro_verified).toBe(true); + expect(outcome.warnings?.some((line) => line.startsWith("model_unverified: "))).toBe(false); + }); + it("records the consult in the bridge ledger like a CLI ask", async () => { const cwd = await mkdtemp(path.join(tmpdir(), "prodex-mcp-consult-")); sendChatGptPromptMock.mockResolvedValueOnce({