Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ All notable changes to this project are documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## Unreleased

### Fixed
- Consult answers came back with `model_used` and `pro_verified` missing, so a Pro consult could not be told apart from one nobody checked. ChatGPT stopped tagging answers with the model that wrote them: measured 2026-11-17, the page had no `data-message-model-slug` anywhere, and neither the answer's buttons nor its More actions menu name a model. prodex reads rendered page content only, so the answering model cannot be read. `model_used` is now `"unknown"` in that case, a `model_unverified` warning says why `pro_verified` is left out, and the new `picker_effort` field (CLI JSON, MCP `pro_consult`, the receipt and a `picker_effort:` line) records the step the picker's effort slider read once prodex had set it, before the prompt was typed. That shows what was selected, not what answered. When ChatGPT names the model again, `model_used` and `pro_verified` are filled from it as before.

## 0.40.52

### Fixed
Expand Down
8 changes: 6 additions & 2 deletions docs/clients.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,12 @@ set this flag, carry it into later calls, or switch chats/keys to evade the chec
Approval is caller attestation, not independent human authentication. Manual CLI
calls and other bridge roots are outside this cooperative MCP guard.

The response exposes `model_used`, `pro_verified`, `continued_from`, `request_id`,
and `request_verified` when available. A ready-to-use `continuation` is only offered
The response exposes `model_used`, `pro_verified`, `picker_effort`, `continued_from`,
`request_id`, and `request_verified` when available. Since November 2026 the ChatGPT
page no longer names the model that wrote an answer, so `model_used` is usually
`"unknown"`, `pro_verified` is left out, and a `model_unverified` warning says so.
`picker_effort` is the step the picker's effort slider read before the prompt was
typed: evidence of what was selected, not of what answered. A ready-to-use `continuation` is only offered
for a saved, request-verified conversation answer (or an approval checkpoint's
already-resolved target). An answer that failed to save or is incomplete does not
invite another automatic turn; report it and resolve the blocker first.
Expand Down
17 changes: 14 additions & 3 deletions src/chatgpt-browser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,11 @@ export interface SendChatGptPromptResult {
modelHints: string[];
/** The model ChatGPT says produced the answer, e.g. "gpt-5-6-pro". */
modelSlug?: string;
/**
* The step the picker's effort slider read once prodex had moved it, before
* the prompt was typed. Evidence of what was selected, not of what answered.
*/
pickerEffort?: string;
/**
* The project id the composer was confirmed bound to before typing, so the
* caller can check the answered thread against where the prompt was aimed
Expand Down Expand Up @@ -4027,7 +4032,8 @@ export function sliderWalkPlan(state: { position?: number; min?: number; max?: n
async function selectModelReasoning(
cdp: CdpConnection,
options: Pick<SendChatGptPromptOptions, "model" | "proMode" | "effort" | "allowSelectionFallback">,
selectionWarnings: string[] = []
selectionWarnings: string[] = [],
selectionEvidence: { effort?: string } = {}
): Promise<void> {
if (!options.model && !options.proMode && !options.effort) return;
// --pro-mode selects a Pro sub-mode, so it is meaningless with a non-Pro
Expand Down Expand Up @@ -4119,7 +4125,8 @@ async function selectModelReasoning(
throw new Error("ChatGPT's model picker would not reopen after the model was chosen.");
}
try {
await selectPowerStepWithGrace(cdp, plan.sliderLabel);
const reached = await selectPowerStepWithGrace(cdp, plan.sliderLabel);
if (reached.effort) selectionEvidence.effort = reached.effort;
} catch (error) {
// Only "this slider has no such step" is the picker declining. A
// slider that will not open, or will not move - the hydrating case,
Expand Down Expand Up @@ -5059,6 +5066,7 @@ export async function sendChatGptPrompt(options: SendChatGptPromptOptions): Prom
let submitButtonFound = false;
let submissionAttempted = false;
const sendWarnings: string[] = [...preflightWarnings];
const selectionEvidence: { effort?: string } = {};
// Anything the page put in front of prodex was answered on the caller's
// behalf. The note is read at return time - there are several return paths -
// so it is folded into the array rather than pushed from each of them.
Expand Down Expand Up @@ -5115,7 +5123,7 @@ export async function sendChatGptPrompt(options: SendChatGptPromptOptions): Prom
}
boundProjectId = await selectProject(cdp, options);
try {
await selectModelReasoning(cdp, options, sendWarnings);
await selectModelReasoning(cdp, options, sendWarnings, selectionEvidence);
} catch (modelError) {
// Pro sub-mode isn't exposed in this UI yet (staged rollout). Pro itself is
// already selected by this point, so degrade to plain Pro with a warning
Expand Down Expand Up @@ -5403,6 +5411,7 @@ export async function sendChatGptPrompt(options: SendChatGptPromptOptions): Prom
title: completed.title,
answer: CHATGPT_NON_TEXT_ANSWER_NOTE,
modelHints: completed.modelHints,
...(selectionEvidence.effort ? { pickerEffort: selectionEvidence.effort } : {}),
...(boundProjectId ? { boundProjectId } : {}),
requestId,
requestVerified: true,
Expand All @@ -5417,6 +5426,7 @@ export async function sendChatGptPrompt(options: SendChatGptPromptOptions): Prom
answer: completed.answer.trim(),
modelHints: completed.modelHints,
...(completed.modelSlug ? { modelSlug: completed.modelSlug } : {}),
...(selectionEvidence.effort ? { pickerEffort: selectionEvidence.effort } : {}),
...(boundProjectId ? { boundProjectId } : {}),
requestId,
requestVerified: true,
Expand All @@ -5436,6 +5446,7 @@ export async function sendChatGptPrompt(options: SendChatGptPromptOptions): Prom
answer: completed.answer.trim(),
modelHints: completed.modelHints,
...(completed.modelSlug ? { modelSlug: completed.modelSlug } : {}),
...(selectionEvidence.effort ? { pickerEffort: selectionEvidence.effort } : {}),
...(boundProjectId ? { boundProjectId } : {}),
requestId,
requestVerified: true,
Expand Down
29 changes: 26 additions & 3 deletions src/cli-pro.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1879,11 +1879,14 @@ export async function runAskProCommand(
if (consult.requestVerified === false && !persistenceWarnings.some((warning) => warning.startsWith("request_unverified:"))) {
persistenceWarnings.push("request_unverified: the saved answer was not verified against its requested user turn. Review the original conversation before continuing.");
}
if (!consult.modelSlug) persistenceWarnings.push(modelUnverifiedWarning(consult.pickerEffort));
const modelUsed = consult.modelSlug ?? "unknown";
// Truncation and other send warnings must be visible at runtime, not
// only inside the persisted receipt: a caller who never opens .bridge
// would otherwise treat a cut-off answer as complete.
for (const warning of persistenceWarnings) io.stderr(warning);
if (consult.modelSlug) io.stderr(`model_used: ${consult.modelSlug}`);
io.stderr(`model_used: ${modelUsed}`);
if (consult.pickerEffort) io.stderr(`picker_effort: ${consult.pickerEffort}`);
if (consult.requestId) io.stderr(`request_id: ${consult.requestId}`);
if (consult.requestVerified !== undefined) io.stderr(`request_verified: ${consult.requestVerified ? "yes" : "no"}`);
// Which conversation this followed, and where the answer actually landed.
Expand Down Expand Up @@ -1930,7 +1933,8 @@ export async function runAskProCommand(
...(Object.keys(selectionMetadata).length > 0 ? { selection: selectionMetadata } : {}),
// What actually answered, straight from ChatGPT's own tag - the
// receipt used to record only what prodex asked for.
...(consult.modelSlug ? { model_used: consult.modelSlug } : {}),
model_used: modelUsed,
...(consult.pickerEffort ? { picker_effort: consult.pickerEffort } : {}),
...(proVerified !== undefined ? { pro_verified: proVerified } : {}),
...(consult.requestId ? { request_id: consult.requestId } : {}),
...(consult.requestVerified !== undefined ? { request_verified: consult.requestVerified } : {}),
Expand Down Expand Up @@ -1998,7 +2002,8 @@ export async function runAskProCommand(
...(consult.requestId ? { request_id: consult.requestId } : {}),
...(consult.requestVerified !== undefined ? { request_verified: consult.requestVerified } : {}),
...(continuedFromTaskId ? { continued_from: continuedFromTaskId } : {}),
...(consult.modelSlug ? { model_used: consult.modelSlug } : {}),
model_used: modelUsed,
...(consult.pickerEffort ? { picker_effort: consult.pickerEffort } : {}),
...(proVerified !== undefined ? { pro_verified: proVerified } : {}),
destination: {
observed: destination.destination,
Expand Down Expand Up @@ -2124,6 +2129,8 @@ export interface BrowserConsultOutcome {
request_id?: string;
request_verified?: boolean;
model_used?: string;
/** The step the picker's effort slider read before the prompt was typed. */
picker_effort?: string;
pro_verified?: boolean;
continued_from?: string;
continuation?: Pick<BrowserConsultInput, "continue_task" | "session_key" | "model" | "effort" | "project">;
Expand Down Expand Up @@ -2977,6 +2984,22 @@ export function proSelectionVerified(selection: {
return /pro/i.test(selection.modelSlug);
}

/**
* Why model_used is "unknown" and pro_verified is missing. ChatGPT stopped
* tagging answers with the model that wrote them (measured 2026-11-17: no
* data-message-model-slug anywhere on the page, and nothing else on screen
* names it), and prodex reads rendered page content only. Leaving both fields
* out without a word read as "not checked" rather than "cannot be checked".
*/
export function modelUnverifiedWarning(pickerEffort: string | undefined): string {
return (
'model_unverified: the ChatGPT page did not name the model that wrote this answer, so model_used is "unknown" and pro_verified is left out.' +
(pickerEffort
? ` Before the prompt was typed, the picker's effort slider read "${pickerEffort}"; that shows what was selected, not what answered.`
: "")
);
}

/**
* A temporary chat is not saved; a project chat is. Entering a project leaves
* temporary mode, so asking for both is asking for two different things, and
Expand Down
2 changes: 1 addition & 1 deletion src/mcp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,7 @@ export function createServer(cwd = process.cwd(), options: CreateMcpServerOption
.boolean()
.optional()
.describe(
"Send even when the requested EFFORT step could not be applied. Off by default, because an answer from a step nobody asked for is usually unusable: a consult asking for Pro that comes back from a lesser model cannot be cited as a Pro review. Pass true only when any answer beats no answer. It does not gate the model axis: a model the picker does not offer is a warning and the send goes out on whatever the composer had - check `model_used` and `pro_verified` in the answer to see what actually replied."
"Send even when the requested EFFORT step could not be applied. Off by default, because an answer from a step nobody asked for is usually unusable: a consult asking for Pro that comes back from a lesser model cannot be cited as a Pro review. Pass true only when any answer beats no answer. It does not gate the model axis: a model the picker does not offer is a warning and the send goes out on whatever the composer had - check `model_used` and `pro_verified` in the answer to see what actually replied. When the page does not name the model, `model_used` is \"unknown\" and `picker_effort` shows the step the picker read before typing."
)
}
},
Expand Down
24 changes: 24 additions & 0 deletions tests/chatgpt-browser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,30 @@ Show more`;
expect(reads).toBeGreaterThan(1);
});

it("returns the step the power slider read once the walk reached it", async () => {
// The answer no longer names its model, so this reading, taken before the
// prompt is typed, is the only evidence of what was selected.
vi.useFakeTimers();
const thread = "https://chatgpt.com/c/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
installFakeChatGptSendCdp(thread, [
{ ...fakeAnswerState(thread, "", false), assistantMessageCount: 0, userMessageCount: 0 },
fakeAnswerState(thread, "correct answer", false)
]);
const base = FakeCdpWebSocket.evaluate;
FakeCdpWebSocket.evaluate = (expression) => {
if (expression === modelButtonRectExpression()) return { ok: true, x: 20, y: 20 };
if (expression === menuOpenExpression() || expression.includes("document.elementFromPoint")) return true;
if (expression === powerSliderPresentExpression()) return true;
if (expression === powerSliderStateExpression()) return { ok: true, position: 4, min: 0, max: 4, effort: "Pro" };
if (expression === focusPowerSliderExpression()) return { ok: true };
return base(expression);
};
const send = sendChatGptPrompt({ port: 19338, prompt: "answer this", effort: "Pro", targetUrl: thread, timeoutMs: 10_000 });
void send.catch(() => undefined);
await vi.advanceTimersByTimeAsync(30_000);
await expect(send).resolves.toMatchObject({ answer: "correct answer", pickerEffort: "Pro" });
});

it.each([
"https://chatgpt.com/c/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"https://chatgpt.com/g/g-p-1234567890123456-project/c/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
Expand Down
7 changes: 4 additions & 3 deletions tests/cli-pro-browser-send.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1572,8 +1572,9 @@ describe("pro browser ask persistence", () => {
task_id: taskId,
thread: "https://chatgpt.com/c/abc",
// The send passed no model flag and this repo has no saved default, so
// the model-selection nudge is recorded alongside the send warning.
warnings: ["model hint observed", expect.stringContaining("model_selection_warning")]
// the model-selection nudge is recorded alongside the send warning, and
// the answer named no model, which is said too.
warnings: ["model hint observed", expect.stringContaining("model_selection_warning"), expect.stringContaining("model_unverified")]
})
);
const result = JSON.parse(await readFile(path.join(cwd, ".bridge", "results", `${taskId}.json`), "utf8")) as {
Expand Down Expand Up @@ -3040,7 +3041,7 @@ describe("pro browser ask model/project selection", () => {
expect(payload.thread).toBe("https://chatgpt.com/c/json");
expect(payload.answer).toBe("json answer");
expect(payload.session_key).toBe("codex-thread-json");
expect(payload.warnings).toEqual([expect.stringContaining("model_selection_warning")]);
expect(payload.warnings).toEqual([expect.stringContaining("model_selection_warning"), expect.stringContaining("model_unverified")]);
});

it("starts an ordinary consult in a fresh chat while keeping the default project", async () => {
Expand Down
51 changes: 51 additions & 0 deletions tests/mcp-consult.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,57 @@ describe("performBrowserConsultForMcp", () => {
expect(outcome.notes.some((note) => note.startsWith("answer_incomplete:"))).toBe(true);
});

// Measured 2026-11-17: ChatGPT stopped tagging answers with the model that
// wrote them, and a Pro consult came back with model_used and pro_verified
// simply missing. The caller could not tell "not checked" from "checked and
// unknowable", so it is said, with the one piece of evidence prodex has.
it("says the answering model is unknown when the page does not name it, and shows what the picker read", async () => {
const cwd = await mkdtemp(path.join(tmpdir(), "prodex-mcp-consult-"));
sendChatGptPromptMock.mockResolvedValueOnce({
url: "https://chatgpt.com/c/mcp-untagged",
title: "ChatGPT",
answer: "untagged answer",
modelHints: [],
pickerEffort: "Pro",
warnings: [],
requestId: "9cb9650622e74a62bd9074c42a311945",
requestVerified: true
});

const outcome = await performBrowserConsultForMcp(cwd, { prompt: "Untagged check", effort: "Pro" });
const { BridgeStore } = await import("../src/store.js");
const receipts = await new BridgeStore(cwd).listReceipts({ kind: "consult_answer_saved", task_id: outcome.task_id });

expect(outcome.model_used).toBe("unknown");
expect(outcome.pro_verified).toBeUndefined();
expect(outcome.picker_effort).toBe("Pro");
const warning = outcome.warnings?.find((line) => line.startsWith("model_unverified: "));
expect(warning).toContain('"Pro"');
expect(receipts[0]?.metadata).toEqual(expect.objectContaining({ model_used: "unknown", picker_effort: "Pro" }));
expect(receipts[0]?.metadata).not.toHaveProperty("pro_verified");
});

it("keeps quiet about the model when the page names it", async () => {
const cwd = await mkdtemp(path.join(tmpdir(), "prodex-mcp-consult-"));
sendChatGptPromptMock.mockResolvedValueOnce({
url: "https://chatgpt.com/c/mcp-tagged",
title: "ChatGPT",
answer: "tagged answer",
modelHints: [],
modelSlug: "gpt-6-pro",
pickerEffort: "Pro",
warnings: [],
requestId: "9cb9650622e74a62bd9074c42a311945",
requestVerified: true
});

const outcome = await performBrowserConsultForMcp(cwd, { prompt: "Tagged check", effort: "Pro" });

expect(outcome.model_used).toBe("gpt-6-pro");
expect(outcome.pro_verified).toBe(true);
expect(outcome.warnings?.some((line) => line.startsWith("model_unverified: "))).toBe(false);
});

it("records the consult in the bridge ledger like a CLI ask", async () => {
const cwd = await mkdtemp(path.join(tmpdir(), "prodex-mcp-consult-"));
sendChatGptPromptMock.mockResolvedValueOnce({
Expand Down
Loading