diff --git a/application/single_app/config.py b/application/single_app/config.py index 77387246d..6cc00241d 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.255" +VERSION = "0.261.256" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/v2_ui/src/components/chat/MessageList.tsx b/application/v2_ui/src/components/chat/MessageList.tsx index 44e257243..c7af94bf4 100644 --- a/application/v2_ui/src/components/chat/MessageList.tsx +++ b/application/v2_ui/src/components/chat/MessageList.tsx @@ -47,6 +47,7 @@ import { OrchestrationPlanCard } from './OrchestrationPlanCard'; import { ElicitationCard } from './ElicitationCard'; import { selectActiveTurn, + selectActiveTurnRunInFlight, selectElicitation, selectPlan, useOrchestrationStore, @@ -1263,11 +1264,27 @@ const MessageBubble = memo(MessageBubbleInner); * A recovered stream is deliberately made to look ordinary once it is flowing again. The * interruption is worth a brief note, but leaving "Reconnecting" under an answer that is * actively arriving reads as a stall, which is the opposite of what is happening. + * + * An orchestrated turn shows one indicator at a time. While it plans, this bubble is the only + * sign of work, and it says "Planning" so the plan card that replaces it reads as the result. While + * an approved plan runs, the plan card already shows its progress, so this bubble draws nothing + * until the answer arrives and then only the answer. Its run notices stay with the finished + * message's reasoning steps. If the card is ever not drawing the run, the bubble shows as usual, + * so a run is never left with no indicator at all. */ function StreamingBubble() { - const { streamingContent, streamingReasoningAdjustments, thoughts, reconnectPhase } = useChatStore(); + const { + streamingContent, + streamingReasoningAdjustments, + thoughts, + reconnectPhase, + orchestrationSurface, + activeConversationId, + } = useChatStore(); const collaborative = useChatStore((state) => state.activeConversationKind === 'collaborative'); const chatWidth = useUiStore((state) => state.chatWidth); + const runCardShowsProgress = useOrchestrationStore((state) => + selectActiveTurnRunInFlight(state, activeConversationId ?? '')); const [showReconnectedNote, setShowReconnectedNote] = useState(false); useEffect(() => { @@ -1282,6 +1299,12 @@ function StreamingBubble() { }, [reconnectPhase]); const connecting = reconnectPhase === 'connecting'; + const runShownOnCard = orchestrationSurface === 'running' && runCardShowsProgress; + const activityLabel = orchestrationSurface === 'planning' ? 'Planning' : 'Thinking'; + + if (runShownOnCard && !streamingContent) { + return null; + } // In a shared conversation the activity line says the request is running, for everyone // in it, so the bubble only appears once there is an answer to show. @@ -1307,7 +1330,7 @@ function StreamingBubble() {

)} - + {!runShownOnCard && } {streamingContent ? ( ) : ( @@ -1321,7 +1344,7 @@ function StreamingBubble() { /> ))} - {connecting ? 'Reconnecting' : 'Thinking'} + {connecting ? 'Reconnecting' : activityLabel} )} diff --git a/application/v2_ui/src/components/chat/OrchestrationPlanCard.tsx b/application/v2_ui/src/components/chat/OrchestrationPlanCard.tsx index 8fb5696f1..d8f6e24ec 100644 --- a/application/v2_ui/src/components/chat/OrchestrationPlanCard.tsx +++ b/application/v2_ui/src/components/chat/OrchestrationPlanCard.tsx @@ -35,12 +35,12 @@ import { import { applyPlanEdits, describePlanner, + describeRunProgress, isPlanRunnable, isPlanTerminal, planBindingIssues, planHasApprovalFloor, planRequiresApproval, - stepRoleLabel, summarizePlan, } from '../../lib/orchestrationPlan'; import { @@ -236,7 +236,9 @@ export function OrchestrationPlanCard({ } // Running: a compact progress line. The full step list is a click away in the drawer, so this - // stays a single row rather than duplicating it. + // stays a single row rather than duplicating it. It is the run's only progress indicator, as + // the streaming bubble stays out of the way until there is an answer, so its status line says + // what the run is doing in words. if (runInFlight || waiting) { const total = summary.step_count; let completed = 0; @@ -249,9 +251,7 @@ export function OrchestrationPlanCard({ completed += 1; } } - const currentStep = editedPlan?.steps.find((step) => stepRuntime[step.step_id]?.status === 'running'); - const progress = waiting ? 'Waiting for results' - : currentStep ? stepRoleLabel(currentStep, true) : null; + const progress = describeRunProgress(editedPlan?.steps ?? [], stepRuntime, waiting); return (
diff --git a/application/v2_ui/src/lib/orchestrationController.ts b/application/v2_ui/src/lib/orchestrationController.ts index f0cb12ad1..68ef2d707 100644 --- a/application/v2_ui/src/lib/orchestrationController.ts +++ b/application/v2_ui/src/lib/orchestrationController.ts @@ -765,8 +765,9 @@ async function executeSavedPlan( const context = turnContexts.get(scopeKey(conversationId, turnId)); // Enter the streaming state for the answer without a second user bubble — the question is - // already in the thread from planning. - useChatStore.getState().beginOrchestrationTurn(conversationId, '', false); + // already in the thread from planning. Taken as the run phase: the plan card shows a run's + // progress, so the streaming bubble stays out of the way until there is an answer to show. + useChatStore.getState().beginOrchestrationTurn(conversationId, '', false, undefined, undefined, 'running'); const controller = new AbortController(); activeControllers.get(conversationId)?.abort(); diff --git a/application/v2_ui/src/lib/orchestrationPlan.ts b/application/v2_ui/src/lib/orchestrationPlan.ts index e925cd2dd..029a7a3d2 100644 --- a/application/v2_ui/src/lib/orchestrationPlan.ts +++ b/application/v2_ui/src/lib/orchestrationPlan.ts @@ -904,6 +904,59 @@ export function stepRoleLabel(step: OrchestrationStep, progress = false): string return ROLE_LABELS[role][progress ? 'progress' : 'label']; } +/** Step statuses after which a step reports nothing more in this run. */ +const SETTLED_STEP_STATUSES: ReadonlySet = new Set([ + 'completed', 'partial', 'skipped', 'failed', 'cancelled', +]); + +/** + * The running plan card's one-line account of what the run is doing, or null for nothing new. + * + * The card is the only progress shown while a plan runs, so this line has to say what is + * happening in words: which kind of work the running step is and what it is called, that the run + * is starting or finishing, or that it is waiting. A bare role label such as "Reasoning" did not + * say what was being reasoned about. The server sends the answer in one piece once every step has + * settled, so the quiet gap before it is named rather than left blank. Disabled steps never run and + * are ignored. Between two steps, which lasts only as long as the executor takes to start the + * next, there is nothing new to say. + */ +export function describeRunProgress( + steps: readonly OrchestrationStep[], + runtime: Readonly>, + waiting = false, +): string | null { + if (waiting) { + return 'Waiting for results'; + } + const enabled = steps.filter((step) => step.enabled); + const statusOf = (step: OrchestrationStep): StepStatus | undefined => runtime[step.step_id]?.status; + const running = enabled.find((step) => statusOf(step) === 'running'); + if (running) { + const role = stepRoleLabel(running, true); + const title = running.title?.trim(); + if (role && title) { + return `${role}: ${title}`; + } + return role || title || 'Running'; + } + if (enabled.some((step) => statusOf(step) === 'waiting')) { + return 'Waiting for results'; + } + if (enabled.length > 0 && enabled.every((step) => { + const status = statusOf(step); + return status !== undefined && SETTLED_STEP_STATUSES.has(status); + })) { + return 'Preparing the answer'; + } + if (enabled.every((step) => { + const status = statusOf(step); + return status === undefined || status === 'pending'; + })) { + return 'Starting'; + } + return null; +} + export interface OrchestrationStepGroup { key: string; label: string | null; diff --git a/application/v2_ui/src/stores/chatStore.ts b/application/v2_ui/src/stores/chatStore.ts index 286a39833..ff27e1ae3 100644 --- a/application/v2_ui/src/stores/chatStore.ts +++ b/application/v2_ui/src/stores/chatStore.ts @@ -217,6 +217,16 @@ export type { ConversationKind }; */ export type ReconnectPhase = 'connecting' | 'reconnected' | null; +/** + * Which part of an orchestrated turn holds the streaming surface. + * + * `planning` is the planner working before a plan or question exists, and `running` is an approved + * plan executing. They are told apart because they draw differently. While a turn plans, the + * streaming bubble is the only sign of progress. A run's progress is already on the plan card, so a + * bubble beside it only repeated it. + */ +export type OrchestrationSurfacePhase = 'planning' | 'running'; + export interface ComposerOptions { /** * The picker's selection key for the chosen model, NOT its deployment name. @@ -352,6 +362,16 @@ interface ChatState { * the first state for the whole reattached stream makes working output look stalled. */ reconnectPhase: ReconnectPhase; + /** + * The orchestration phase holding the open conversation's `streaming` flag, or null when the + * flag belongs to an ordinary chat stream or nothing is streaming. + * + * `streaming` alone cannot say whose it is. A chat message can be sent while an orchestration + * run waits or is being reconciled in the same conversation, and that reply must still show + * Thinking, so the owner is recorded when the flag is taken rather than guessed from the + * orchestration store. + */ + orchestrationSurface: OrchestrationSurfacePhase | null; /** Right-hand drawer state. Null means closed. */ drawerMode: DrawerMode; @@ -453,7 +473,9 @@ interface ChatState { * `beginOrchestrationTurn` returns the optimistic user bubble's id so the controller can hand * it back on completion for reconciliation with the server's persisted id. Passing * `addUserMessage: false` re-enters the thinking state for a re-plan without adding a second - * bubble, because the user's question is already in the thread from the first plan. + * bubble, because the user's question is already in the thread from the first plan. `phase` + * says which part of the turn takes the surface: a plan by default, or `running` for an + * approved plan, whose progress the plan card shows instead of the streaming bubble. */ beginOrchestrationTurn: ( conversationId: string, @@ -461,6 +483,7 @@ interface ChatState { addUserMessage?: boolean, turnId?: string, promptInfo?: Json | null, + phase?: OrchestrationSurfacePhase, ) => string; pushOrchestrationThought: (conversationId: string, event: RunStreamEvent) => void; pushOrchestrationContent: (conversationId: string, accumulated: string) => void; @@ -873,7 +896,7 @@ let streamingConversationId: string | null = null; let streamingConversationKind: ConversationKind = 'personal'; /** - * Conversations whose orchestration turn currently holds the streaming surface. + * Conversations whose orchestration turn currently holds the streaming surface, and in which phase. * * A plan or run is driven by its own controller, not `activeStreamController`, so * `detachActiveStream` knows nothing about it. The turn takes the shared `streaming` flag in @@ -883,9 +906,10 @@ let streamingConversationKind: ConversationKind = 'personal'; * Stop with nothing to stop, and refused to send until the page was reloaded. * * Kept per conversation, whatever is on screen, so leaving can drop the flag and reopening a - * conversation whose turn is still running in this tab can put it back. + * conversation whose turn is still running in this tab can put it back. The phase comes back with + * it, so a reopened run still shows its progress on the plan card alone. */ -const orchestrationSurfaces = new Set(); +const orchestrationSurfaces = new Map(); /** * Bumped each time the reader starts a new chat or opens a conversation. @@ -1550,6 +1574,7 @@ async function resumeChatStream(conversationId: string): Promise { set({ streaming: true, + orchestrationSurface: null, streamingContent: '', thoughts: [], streamingReasoningAdjustments: [], @@ -2069,6 +2094,7 @@ export const useChatStore = create((set, get) => ({ streamError: null, streamAuthUrl: null, reconnectPhase: null, + orchestrationSurface: null, drawerMode: null, metadata: null, @@ -2176,6 +2202,7 @@ export const useChatStore = create((set, get) => ({ // thoughts and its settle land here as usual. A chat stream is picked back up by // `resumeChatStream` below. streaming: conversationId ? orchestrationSurfaces.has(conversationId) : false, + orchestrationSurface: conversationId ? orchestrationSurfaces.get(conversationId) ?? null : null, streamingContent: '', thoughts: [], streamingReasoningAdjustments: [], @@ -2389,6 +2416,7 @@ export const useChatStore = create((set, get) => ({ // once its conversation is off screen, so the new chat kept the old turn's Thinking // state and a Stop button with nothing to stop. The turn itself keeps running. streaming: false, + orchestrationSurface: null, streamingContent: '', thoughts: [], streamingReasoningAdjustments: [], @@ -3182,6 +3210,7 @@ export const useChatStore = create((set, get) => ({ }, ], streaming: true, + orchestrationSurface: null, streamingContent: '', thoughts: [], streamingReasoningAdjustments: [], @@ -3252,12 +3281,19 @@ export const useChatStore = create((set, get) => ({ } }, - beginOrchestrationTurn: (conversationId, text, addUserMessage = true, turnId, promptInfo) => { + beginOrchestrationTurn: ( + conversationId, + text, + addUserMessage = true, + turnId, + promptInfo, + phase = 'planning', + ) => { const trimmed = text.trim(); const pendingUserMessageId = addUserMessage ? `pending-user-${Date.now()}` : ''; // Recorded whether or not the conversation is on screen, so opening it while this turn // is still in flight shows it working. - orchestrationSurfaces.add(conversationId); + orchestrationSurfaces.set(conversationId, phase); // Guarded on the open conversation, exactly like sendMessage's optimistic write: a run // started here keeps going after the reader opens another thread, and its question must // not appear inside that other thread. @@ -3287,6 +3323,7 @@ export const useChatStore = create((set, get) => ({ ] : state.messages, streaming: true, + orchestrationSurface: phase, analysisTurnRevision: state.analysisContextRevision, streamingContent: '', thoughts: [], @@ -3353,6 +3390,7 @@ export const useChatStore = create((set, get) => ({ // the next turn's streaming bubble. set({ streaming: false, + orchestrationSurface: null, streamingContent: '', thoughts: [], streamingReasoningAdjustments: [], @@ -3364,6 +3402,7 @@ export const useChatStore = create((set, get) => ({ if (outcome.status === 'failed' && !('event' in outcome)) { set({ streaming: false, + orchestrationSurface: null, streamingContent: '', reconnectPhase: null, streamError: outcome.error, @@ -3392,6 +3431,7 @@ export const useChatStore = create((set, get) => ({ ] : state.messages, streaming: false, + orchestrationSurface: null, streamingContent: '', reconnectPhase: null, })); @@ -3462,6 +3502,7 @@ export const useChatStore = create((set, get) => ({ finalMessage as CollaborationMessage, ), streaming: false, + orchestrationSurface: null, streamingContent: '', reconnectPhase: null, streamError: null, @@ -3498,8 +3539,9 @@ export const useChatStore = create((set, get) => ({ return; } // The turn's claim on the streaming surface follows it to the id the server named. - if (conversationChanged && orchestrationSurfaces.delete(fromConversationId)) { - orchestrationSurfaces.add(toConversationId); + const surfacePhase = orchestrationSurfaces.get(fromConversationId); + if (conversationChanged && surfacePhase && orchestrationSurfaces.delete(fromConversationId)) { + orchestrationSurfaces.set(toConversationId, surfacePhase); } set((state) => { const messages = state.messages.map((message) => { @@ -3709,6 +3751,7 @@ export const useChatStore = create((set, get) => ({ } set({ streaming: true, + orchestrationSurface: null, streamingContent: '', thoughts: [], streamingReasoningAdjustments: [], @@ -3765,6 +3808,7 @@ export const useChatStore = create((set, get) => ({ } set({ streaming: true, + orchestrationSurface: null, streamingContent: '', thoughts: [], streamingReasoningAdjustments: [], diff --git a/application/v2_ui/src/stores/orchestrationStore.ts b/application/v2_ui/src/stores/orchestrationStore.ts index f929c1290..19799ec23 100644 --- a/application/v2_ui/src/stores/orchestrationStore.ts +++ b/application/v2_ui/src/stores/orchestrationStore.ts @@ -1423,6 +1423,28 @@ export function selectInFlightForConversation( ); } +/** + * Whether the conversation's inline card is drawing a run that is in flight. + * + * True exactly when `ActiveOrchestrationCard` draws the plan card's running row: the active turn + * has a plan, no question supersedes it, and a run for that turn is in flight. The streaming bubble + * reads this so a run's progress appears once, on the card, and so the bubble comes back if the + * card is ever not drawing the run. A boolean, so subscribers re-render only when it flips. + */ +export function selectActiveTurnRunInFlight( + state: OrchestrationState, + conversationId: string, +): boolean { + const turnId = selectActiveTurn(state, conversationId); + if (!turnId || selectElicitation(state, conversationId, turnId) !== null + || selectPlan(state, conversationId, turnId) === null) { + return false; + } + return Object.values(state.inFlight).some( + (run) => run.conversationId === conversationId && run.turnId === turnId, + ); +} + /** How many runs are in flight for a conversation. Drives a row indicator. */ export function selectInFlightCount( state: OrchestrationState, diff --git a/docs/explanation/features/CHAT_ORCHESTRATION.md b/docs/explanation/features/CHAT_ORCHESTRATION.md index 673a3fc41..08cf2ea80 100644 --- a/docs/explanation/features/CHAT_ORCHESTRATION.md +++ b/docs/explanation/features/CHAT_ORCHESTRATION.md @@ -1,6 +1,6 @@ # Chat Orchestration -**Version: 0.261.192** (tracked in `application/single_app/config.py`) +**Version: 0.261.256** (tracked in `application/single_app/config.py`) **Implemented in version: 0.261.086** **Knowledge phase added in version: 0.261.089** @@ -24,6 +24,7 @@ **Deliverables contract and generated images in files implemented in version: 0.261.138** **Gather / Reason / Render made the only orchestration contract in version: 0.261.139** **Photorealistic planned images and reference images implemented in version: 0.261.192** +**One progress indicator per phase (Planning label, run progress on the plan card only) fixed in version: 0.261.256** ## Overview @@ -1137,11 +1138,23 @@ and `orchestration_synthesis`, each carrying `activity.lane_key = "orchestration The client's orchestration lane claims these steps, so they are never counted as agent or tabular work, but it draws no progress card. The planner's steps sit under the reasoning -toggle next to **Thinking**. A run's progress is on the plan card, which counts completed -steps and names the running one, and in the drawer. A card on top repeated both, and during -planning it stayed at 45% until the plan replaced it. See the +toggle next to **Planning**. A run's progress is on the plan card, which counts completed +steps, and in the drawer. A card on top repeated both, and during planning it stayed at 45% +until the plan replaced it. See the [duplicate progress card fix](../fixes/ORCHESTRATION_DUPLICATE_PROGRESS_CARD_FIX.md). +An orchestrated turn shows one progress indicator at a time. The chat store records which +part of the turn holds the streaming surface: `planning` while the planner works and +`running` while an approved plan executes. While planning, the streaming bubble says +**Planning**. While a plan runs, the bubble draws nothing until the answer arrives, and the +plan card's status line is the only indicator. It names the work in progress: "Starting", the +running step's kind of work and title (for example "Gathering: Read quarterly reports"), +"Waiting for results", and "Preparing the answer" once every step has settled. The server sends +the answer in one piece after the last step, so the run's notices are kept with the finished +answer's reasoning steps rather than in a live toggle. Any other stream, such as a chat reply +sent while a run waits, still shows **Thinking**. See the +[duplicate Thinking indicator fix](../fixes/ORCHESTRATION_RUN_DUPLICATE_THINKING_INDICATOR_FIX.md). + Execution terminal events also retain the actual outcome, safe failure information, and attempt/recovery identifiers. A final explanatory message does not imply that every step succeeded. The same information is saved with the @@ -1375,7 +1388,9 @@ to the front. | `functional_tests/test_orchestration_conversation_context_routes.py` | New and existing conversations across HTTP/SSE planning and execution, all approval modes, null clarifications, bounded recovery, model selection and attribution, revocation, completion failures, stream cleanup, stale sources and legacy cutoffs | | `ui_tests/test_v2_orchestration_conversation_context.py` | Matching clarification/model transport, cancellation, original-turn continuity, all approval modes, visible answer model names and navigation | | `functional_tests/test_v2_orchestration_progress_lane.mjs` | Planner, plan-ready, saved-memory and reasoning-adjustment steps stay in the orchestration lane, which draws no progress card; an agent hand-off inside an orchestrated turn stays there; tabular and agent lanes keep their cards | -| `ui_tests/test_v2_orchestration_streaming_bubble.py` | While planning, the streaming bubble shows only the reasoning toggle and **Thinking**; no orchestration progress card appears during a run or in a finished answer's reasoning; a tabular turn still draws its card | +| `ui_tests/test_v2_orchestration_streaming_bubble.py` | While planning, the streaming bubble shows only the reasoning toggle and **Planning**; while a plan runs, manually or auto-approved, only the plan card shows, its status line naming each step and then "Preparing the answer", and **Thinking** never appears; the run's notice stays with the finished answer's reasoning; a stream that is not the run's still shows **Thinking**; no orchestration progress card appears; a tabular turn still draws its card | +| `functional_tests/test_v2_orchestration_run_progress_status.mjs` | The running plan card's status line: starting, a running step's kind of work and title with fallbacks, the gap between steps, every step settled including failed and skipped ones, disabled steps, and waiting runs and steps | +| `functional_tests/test_v2_orchestration_streaming_surface.mjs` | The real controller and stores: the planner holds the streaming surface as `planning` and an approved run as `running`; an auto-approved plan hands it straight to the run; a chat reply sent while a run waits is not mistaken for the run; the phase survives leaving, returning and server re-keying; every write that takes the streaming flag names its owner | Research-selection evaluation distinguishes contract coverage from model behaviour. A mocked plan proves that the application preserves an allowed choice; it does not prove diff --git a/docs/explanation/fixes/ORCHESTRATION_DUPLICATE_PROGRESS_CARD_FIX.md b/docs/explanation/fixes/ORCHESTRATION_DUPLICATE_PROGRESS_CARD_FIX.md index 71e4c8631..e2bddf56a 100644 --- a/docs/explanation/fixes/ORCHESTRATION_DUPLICATE_PROGRESS_CARD_FIX.md +++ b/docs/explanation/fixes/ORCHESTRATION_DUPLICATE_PROGRESS_CARD_FIX.md @@ -93,3 +93,6 @@ card instead. - [Chat Orchestration](../features/CHAT_ORCHESTRATION.md), under "Stream events" - [V2 Tabular Analysis](../features/V2_TABULAR_ANALYSIS.md), under "Progress lanes" +- [Orchestration Duplicate Thinking Indicator Fix](ORCHESTRATION_RUN_DUPLICATE_THINKING_INDICATOR_FIX.md) + (0.261.256), which removed the **Thinking** bubble that was still drawn beside the plan card + during a run and relabelled the planning bubble **Planning** diff --git a/docs/explanation/fixes/ORCHESTRATION_RUN_DUPLICATE_THINKING_INDICATOR_FIX.md b/docs/explanation/fixes/ORCHESTRATION_RUN_DUPLICATE_THINKING_INDICATOR_FIX.md new file mode 100644 index 000000000..ab1595b08 --- /dev/null +++ b/docs/explanation/fixes/ORCHESTRATION_RUN_DUPLICATE_THINKING_INDICATOR_FIX.md @@ -0,0 +1,164 @@ +# Orchestration Duplicate Thinking Indicator Fix (v0.261.256) + +## Issue + +Once an orchestration plan started running, the V2 chat showed two progress indicators for the +same work: + +- **Streaming bubble:** "Thinking". When a run notice such as a saved-memory warning arrived, a + "1 reasoning step" toggle appeared above it. +- **Plan card running row:** a spinner, the plan's intent, a step count such as `0/1`, + **Review**, and a status line that said only "Reasoning". + +Neither said what the run was actually doing. The planning phase before them also said +"Thinking", which didn't match the plan card that replaced it. + +Fixed in version: **0.261.256**, tracked in `application/single_app/config.py`. + +## Root cause + +A run is driven by the orchestration controller, but it borrows the chat store's shared +`streaming` flag. That flag routes **Stop** to the run, keeps the composer from sending into a busy +turn, and collects the run's thoughts for the finished answer. `executeSavedPlan` took the flag +through `beginOrchestrationTurn`, and `MessageList` drew `StreamingBubble` whenever `streaming` was +true. The store recorded only that something was streaming, not that it was the run, so the bubble +could not step aside for the card. + +The bubble had nothing to add while the run worked. The server never streams a run's answer. +`functions_orchestration_execution.py` sends `orchestration_step` frames and the occasional notice +thought while steps run, then the whole answer in one `content` frame immediately before `done`. +For the entire run the bubble therefore showed only "Thinking". The +[duplicate progress card fix](ORCHESTRATION_DUPLICATE_PROGRESS_CARD_FIX.md) in 0.261.204 removed +the orchestration lane's card for the same reason, but left this placeholder. + +The owner can't be inferred from the orchestration store. The composer only blocks sending while +`streaming` is true, so a chat reply can be sent while a run waits for a durable result or is being +reconciled after a dropped connection. That run is still in flight on its card. Hiding the bubble +whenever a run was in flight would also have hidden the chat reply's **Thinking**. + +## Technical details + +### Files modified + +- `application/v2_ui/src/stores/chatStore.ts` + - Adds `OrchestrationSurfacePhase` (`planning` or `running`) and `orchestrationSurface`. This + state records which phase holds the open conversation's `streaming` flag, or null for a chat + stream. + - `orchestrationSurfaces` is now a `Map` from conversation id to phase. Leaving and reopening a + conversation, and a server re-key in `reassignOrchestrationTurn`, keep the phase. + - `beginOrchestrationTurn` takes an optional trailing `phase` that defaults to `planning`, so + existing callers are unchanged. Every visible branch of `settleOrchestrationTurn` releases it, + `selectConversation` restores it with `streaming`, and `startNewConversation` clears it. + - `sendMessage`, `resumeChatStream`, `retryMessage` and `editMessage` set + `orchestrationSurface: null` when they take the flag. +- `application/v2_ui/src/lib/orchestrationController.ts`: `executeSavedPlan` takes the surface as + `running`. Stop, the composer lock and thought collection are unchanged. +- `application/v2_ui/src/stores/orchestrationStore.ts`: adds `selectActiveTurnRunInFlight`. It is + true exactly when `ActiveOrchestrationCard` draws the plan card's running row: the active turn has + a plan, no pending question, and a run in flight. +- `application/v2_ui/src/components/chat/MessageList.tsx`: `StreamingBubble` changes in two ways. + - **Running:** while the run holds the surface and the card is drawing it, the bubble draws + nothing until the answer arrives, and then shows only the answer. If the card is not drawing + the run, the bubble shows as usual, so a run always has an indicator. + - **Planning:** the label is **Planning** instead of **Thinking**. +- `application/v2_ui/src/lib/orchestrationPlan.ts`: adds `describeRunProgress`, which writes the + running card's status line. +- `application/v2_ui/src/components/chat/OrchestrationPlanCard.tsx`: the running row's status line + comes from `describeRunProgress`. The row's layout, step count and **Review** are unchanged. +- `application/single_app/config.py`: version bumped to `0.261.256`. + +### Status line + +The first matching row wins. + +| Run state | Status line | +|---|---| +| The plan is waiting | Waiting for results | +| A step is running | Its kind of work and title, for example "Gathering: Read quarterly reports" | +| A step is waiting | Waiting for results | +| Every enabled step is completed, partial, skipped, failed or cancelled | Preparing the answer | +| No enabled step has reported yet | Starting | +| Between two steps | No line | + +Disabled steps are ignored. A running step without a role shows its title, and one without a title +shows its role. + +### Tests + +- `functional_tests/test_v2_orchestration_run_progress_status.mjs`, run with `node`, executes + `describeRunProgress` for the status line cases above, including: + - role labels for gather, reason and render steps, with fallbacks; + - several steps running at once; + - a run whose only step failed; + - disabled steps; + - waiting runs and waiting steps. +- `functional_tests/test_v2_orchestration_streaming_surface.mjs`, run with `node`, drives the real + controller and stores against held SSE streams. It checks that: + - the planner holds the surface as `planning` and an approved run holds it as `running`; + - an auto-approved plan hands the surface straight to the run; + - a chat reply sent while a run waits takes the flag as a chat stream, with the run still on its + card; + - the phase survives leaving, returning and a server re-key; + - the run's notice reaches the finished answer's reasoning steps; + - every write that takes `streaming` names its owner. +- `ui_tests/test_v2_orchestration_streaming_bubble.py` runs the production controller, stores, SSE + reader, `MessageList` and plan card in Chromium. It checks that: + - while planning, only the toggle and **Planning** show; + - for a manually approved run, only the card shows, and its status line moves through + "Starting", "Gathering: Read quarterly reports", "Gathering: Investigate context", + "Reasoning: Write the comparison" and "Preparing the answer"; + - a mutation observer confirms **Thinking** was never drawn during the turn; + - an auto-approved plan goes from **Planning** straight to the run card; + - the finished answer's "1 reasoning step" toggle still reveals the run notice; + - a stream that is not the run's shows **Thinking** beside the card; + - a tabular turn keeps its progress card. + + Before the fix, the four orchestration tests failed and the tabular test passed. +- `ui_tests/test_v2_new_chat_reset.py` now looks for **Planning** while a turn plans and for the + running card while a plan runs. A new chat must show neither, nor **Thinking**. +- `ui_tests/test_v2_reasoning_controls.py` scopes its run-correction status locator to the + reasoning notice, because the running card's own status line ("Starting") is also a status. +- `functional_tests/test_v2_new_chat_reset.py` and `functional_tests/test_v2_stream_reconnect.py` + check the new map bookkeeping and label expression. Their intent is unchanged. + +## Impact + +- **Planning:** the bubble shows the reasoning-step toggle and **Planning**. +- **Running:** the plan card alone shows progress, with a status line that names the work. **Stop** + and the locked composer behave as before. +- **Finished:** the answer shows as before. Notices the run raised are under its reasoning steps. +- **Chat:** chat streams are unchanged, including a reply sent while a run waits in the same + conversation. +- **Server:** unchanged. Stream frames and saved messages are the same. + +## Validation + +- **Before:** after **Approve**, the thread showed "Thinking" in a bubble above a card reading + "Generate and evaluate potential app names… 0/1 Review" and "Reasoning". +- **After:** the thread shows only the card, with a status line such as "Reasoning: Draft candidate + names", then the answer. +- `npm --prefix .\application\v2_ui run typecheck` and the production `vite build` pass. +- Both new Node tests pass, as do `functional_tests/test_v2_orchestration_progress_lane.mjs`, + `test_v2_new_chat_reset.py`, `test_v2_stream_reconnect.py`, `test_v2_stream_leave_without_cancel.py` + and `test_v2_prompt_composer_card.py`, which calls `beginOrchestrationTurn` with positional + arguments. +- These UI suites pass: + - `ui_tests/test_v2_orchestration_streaming_bubble.py` + - `ui_tests/test_v2_new_chat_reset.py` + - `ui_tests/test_v2_orchestration_planning_retry.py` + - `ui_tests/test_v2_orchestration_plan_card.py`, run with `python` +- The wider regression sweep also passes. Run on their own, each of these suites passes: + - approval persistence, auto open, conversation context, model picker, outputs, recovery, + dependency plans, actions, and workflow run links and cards; + - reasoning controls, the elicitation and prompt composers, and the admin orchestration + settings. + + The composer, drawer views, elicitation, pending approval resume, plan editing and run + hydration suites pass when run with `python `, as they are written to be. + +## Related + +- [Chat Orchestration](../features/CHAT_ORCHESTRATION.md), under "Stream events" +- [Orchestration Duplicate Progress Card Fix](ORCHESTRATION_DUPLICATE_PROGRESS_CARD_FIX.md) +- [V2 New Chat Reset Fix](V2_NEW_CHAT_RESET_FIX.md), which introduced the per-conversation surface + bookkeeping this fix extends diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 8ca0709de..1d018486b 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,22 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.256)** + +#### Bug Fixes + +* **One Progress Indicator While Orchestrate Runs a Plan** + * Once a plan started running, the V2 chat showed **Thinking** in a reply bubble above the plan card, which was already showing the run's progress. The plan card is now the only indicator while a plan runs, and the answer appears when it arrives. Run notices, such as a saved-memory warning, are still under the finished answer's reasoning steps. + * A chat reply sent while a plan waits in the same conversation still shows **Thinking**, and **Stop** works as before. + * (Ref: `MessageList.tsx`, `chatStore.ts` `orchestrationSurface`, `executeSavedPlan`, `selectActiveTurnRunInFlight`, `test_v2_orchestration_streaming_bubble.py`, `test_v2_orchestration_streaming_surface.mjs`, [Orchestration Duplicate Thinking Indicator Fix](fixes/ORCHESTRATION_RUN_DUPLICATE_THINKING_INDICATOR_FIX.md)) + +#### User Interface Enhancements + +* **Orchestrate Says What It's Doing** + * While Orchestrate plans a question, the reply bubble now says **Planning** instead of **Thinking**. + * The running plan card's status line names the work instead of a bare role. It shows "Starting", then each step's kind of work and title, such as "Gathering: Read quarterly reports", then "Preparing the answer" once every step has settled. "Waiting for results" is unchanged. + * (Ref: `describeRunProgress` in `orchestrationPlan.ts`, `OrchestrationPlanCard.tsx`, `test_v2_orchestration_run_progress_status.mjs`, [Chat Orchestration](features/CHAT_ORCHESTRATION.md#stream-events)) + ### **(v0.261.253)** #### Bug Fixes diff --git a/functional_tests/test_v2_new_chat_reset.py b/functional_tests/test_v2_new_chat_reset.py index 28c7ec58c..478146c0c 100644 --- a/functional_tests/test_v2_new_chat_reset.py +++ b/functional_tests/test_v2_new_chat_reset.py @@ -3,7 +3,7 @@ """ Functional test for starting a new chat in V2 while the open conversation is busy. -Version: 0.261.226 +Version: 0.261.256 Implemented in: 0.261.226 Clicking New chat while an orchestration turn was planning or running kept the old turn's @@ -111,12 +111,12 @@ def test_orchestration_turns_record_their_streaming_surface_whatever_is_on_scree print("Testing the orchestration surface bookkeeping...") store = _store() - assert "const orchestrationSurfaces = new Set();" in store, ( + assert "const orchestrationSurfaces = new Map();" in store, ( "The store must track which conversations' orchestration turns hold the streaming surface" ) begin = _store_action_body(store, "beginOrchestrationTurn") - claim = begin.index("orchestrationSurfaces.add(conversationId);") + claim = begin.index("orchestrationSurfaces.set(conversationId, phase);") assert claim < begin.index("if (get().activeConversationId === conversationId) {"), ( "beginOrchestrationTurn must record the surface before its on-screen guard, so a turn " "started out of sight still shows as working when its conversation is opened" @@ -131,10 +131,11 @@ def test_orchestration_turns_record_their_streaming_surface_whatever_is_on_scree reassign = _store_action_body(store, "reassignOrchestrationTurn") assert re.search( - r"if \(conversationChanged && orchestrationSurfaces\.delete\(fromConversationId\)\) \{\s*" - r"orchestrationSurfaces\.add\(toConversationId\);", + r"const surfacePhase = orchestrationSurfaces\.get\(fromConversationId\);\s*" + r"if \(conversationChanged && surfacePhase && orchestrationSurfaces\.delete\(fromConversationId\)\) \{\s*" + r"orchestrationSurfaces\.set\(toConversationId, surfacePhase\);", reassign, - ), "A turn re-keyed to the server's conversation id must take its surface with it" + ), "A turn re-keyed to the server's conversation id must take its surface, and its phase, with it" print("Orchestration surface bookkeeping test passed!") return True diff --git a/functional_tests/test_v2_orchestration_run_progress_status.mjs b/functional_tests/test_v2_orchestration_run_progress_status.mjs new file mode 100644 index 000000000..edade661e --- /dev/null +++ b/functional_tests/test_v2_orchestration_run_progress_status.mjs @@ -0,0 +1,105 @@ +// test_v2_orchestration_run_progress_status.mjs +// Version: 0.261.256 +// Implemented in: 0.261.256 +// Executes the real describeRunProgress (lib/orchestrationPlan.ts), which writes the running plan +// card's status line. While a plan runs, the card is the only progress indicator: the streaming +// bubble no longer draws "Thinking" beside it. The line therefore says what the run is doing in +// words, not just a role label: starting, the kind of work and title of the running step, waiting, +// or preparing the answer once every step has settled. + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import './test_support/tsResolve.mjs'; + +// The repository resolver must be registered before extensionless TypeScript imports load. +const { describeRunProgress } = await import('../application/v2_ui/src/lib/orchestrationPlan.ts'); + +/** A plan step with only the fields the status line reads. */ +function step(stepId, { title = `Step ${stepId}`, role = 'reason', enabled = true } = {}) { + return { step_id: stepId, title, role, enabled }; +} + +/** A step runtime map, shaped as orchestration_step frames leave it in the store. */ +function runtime(statuses) { + return Object.fromEntries( + Object.entries(statuses).map(([stepId, status]) => [stepId, { status, summary: '' }]), + ); +} + +test('a run with no step reported yet is starting', () => { + const steps = [step('a'), step('b')]; + assert.equal(describeRunProgress(steps, {}), 'Starting'); + assert.equal(describeRunProgress(steps, runtime({ a: 'pending', b: 'pending' })), 'Starting'); +}); + +test('a running step is named by its kind of work and its title', () => { + const cases = [ + ['gather', 'Search the quarterly reports', 'Gathering: Search the quarterly reports'], + ['reason', 'Draft candidate names', 'Reasoning: Draft candidate names'], + ['render', 'Build the comparison chart', 'Rendering: Build the comparison chart'], + ]; + for (const [role, title, expected] of cases) { + const steps = [step('a', { role, title })]; + assert.equal(describeRunProgress(steps, runtime({ a: 'running' })), expected, role); + } +}); + +test('a running step without a role or a title falls back to what is known', () => { + assert.equal( + describeRunProgress([step('a', { role: null, title: 'Draft candidate names' })], runtime({ a: 'running' })), + 'Draft candidate names', + ); + assert.equal( + describeRunProgress([step('a', { role: 'reason', title: ' ' })], runtime({ a: 'running' })), + 'Reasoning', + ); + assert.equal( + describeRunProgress([step('a', { role: null, title: '' })], runtime({ a: 'running' })), + 'Running', + ); +}); + +test('the first running step in plan order is named when several run at once', () => { + const steps = [ + step('a', { role: 'gather', title: 'Search the reports' }), + step('b', { role: 'gather', title: 'Search the web' }), + ]; + assert.equal( + describeRunProgress(steps, runtime({ a: 'running', b: 'running' })), + 'Gathering: Search the reports', + ); +}); + +test('between two steps there is nothing new to say', () => { + const steps = [step('a'), step('b')]; + assert.equal(describeRunProgress(steps, runtime({ a: 'completed' })), null); + assert.equal(describeRunProgress(steps, runtime({ a: 'completed', b: 'pending' })), null); +}); + +test('once every enabled step has settled the run is preparing the answer', () => { + const steps = [step('a'), step('b'), step('c'), step('d'), step('e')]; + assert.equal( + describeRunProgress(steps, runtime({ + a: 'completed', b: 'partial', c: 'skipped', d: 'failed', e: 'cancelled', + })), + 'Preparing the answer', + ); + // A run whose only step failed is finishing, not starting. + assert.equal(describeRunProgress([step('a')], runtime({ a: 'failed' })), 'Preparing the answer'); +}); + +test('disabled steps never run and are ignored', () => { + const steps = [step('a'), step('b', { enabled: false })]; + assert.equal(describeRunProgress(steps, runtime({ a: 'completed' })), 'Preparing the answer'); + assert.equal(describeRunProgress(steps, runtime({ b: 'running' })), 'Starting'); +}); + +test('a waiting run or step says so, and a running step is named first', () => { + const steps = [step('a', { title: 'Draft candidate names' }), step('b')]; + assert.equal(describeRunProgress(steps, runtime({ a: 'running' }), true), 'Waiting for results'); + assert.equal(describeRunProgress(steps, runtime({ a: 'completed', b: 'waiting' })), 'Waiting for results'); + assert.equal( + describeRunProgress(steps, runtime({ a: 'running', b: 'waiting' })), + 'Reasoning: Draft candidate names', + ); +}); diff --git a/functional_tests/test_v2_orchestration_streaming_surface.mjs b/functional_tests/test_v2_orchestration_streaming_surface.mjs new file mode 100644 index 000000000..77791b0e7 --- /dev/null +++ b/functional_tests/test_v2_orchestration_streaming_surface.mjs @@ -0,0 +1,320 @@ +// test_v2_orchestration_streaming_surface.mjs +// Version: 0.261.256 +// Implemented in: 0.261.256 +// Executes the real chat store, orchestration store and orchestration controller to check who +// owns the streaming surface during an orchestrated turn. While a plan runs, the plan card is the +// only progress indicator, so the streaming bubble needs to know that the surface belongs to a +// run, not just that something is streaming. The planner takes the surface as `planning`, an +// approved run takes it as `running`, and every chat stream takes it as null. The phase follows +// the turn when the reader leaves and returns, and when the server re-keys the conversation. A +// chat reply sent while a run waits in the same conversation must not be mistaken for the run. +// Only HTTP is simulated: plan, run and chat streams are held open and fed server-shaped frames. + +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; +import './test_support/tsResolve.mjs'; + +// The repository resolver must be registered before extensionless TypeScript imports load. +const { useChatStore } = await import('../application/v2_ui/src/stores/chatStore.ts'); +const { selectActiveTurnRunInFlight, useOrchestrationStore } = await import( + '../application/v2_ui/src/stores/orchestrationStore.ts' +); +const { approveAndRunPlan, startOrchestrationPlan } = await import( + '../application/v2_ui/src/lib/orchestrationController.ts' +); + +const PLAN_PATH = '/api/v2/orchestration/plan'; +const RUN_PATH = '/api/v2/orchestration/run'; +const ANSWER = 'Revenue rose 12% between the two quarters.'; +const NOTICE = 'Saved facts could not be searched. Any available instruction memories are still included.'; + +const chat = () => useChatStore.getState(); +const runCardShowsProgress = (conversationId) => + selectActiveTurnRunInFlight(useOrchestrationStore.getState(), conversationId); + +function openConversation(conversationId) { + useChatStore.setState({ + ...useChatStore.getInitialState(), + activeConversationId: conversationId, + activeConversationKind: 'personal', + conversations: [{ id: conversationId, title: 'Quarterly comparison' }], + }, true); +} + +/** + * Replace fetch with streams the test feeds by hand. + * + * Every request opens a held stream keyed by its path, so each intermediate state can be read + * before the next frame lands. A path the test never feeds simply stays pending. + */ +function holdStreams() { + const encoder = new TextEncoder(); + const streams = new Map(); + const waiters = new Map(); + const original = globalThis.fetch; + globalThis.fetch = async (input, init = {}) => { + const raw = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + const path = new URL(raw, 'http://localhost').pathname; + const body = new ReadableStream({ + start(controller) { + streams.set(path, controller); + }, + }); + const waiter = waiters.get(path); + if (waiter) { + waiters.delete(path); + waiter(typeof init.body === 'string' ? JSON.parse(init.body) : null); + } + return new Response(body, { status: 200, headers: { 'Content-Type': 'text/event-stream' } }); + }; + return { + opened(path) { + return new Promise((resolve) => waiters.set(path, resolve)); + }, + emit(path, event) { + streams.get(path).enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`)); + }, + end(path) { + try { + streams.get(path).close(); + } catch { + // The reader already stopped at the terminal frame. + } + }, + restore() { + globalThis.fetch = original; + }, + }; +} + +/** A single-step plan in planner contract 2, the only shape normalizePlan accepts. */ +function makePlan(conversationId, turnId, mode = 'manual') { + return { + plan_id: `plan-${conversationId}`, + run_id: `run-${conversationId}`, + turn_id: turnId, + conversation_id: conversationId, + user_id: 'tester', + revision: 0, + planner_contract_version: 2, + intent: { summary: 'Compare the quarterly reports.', complexity: 'simple' }, + steps: [{ + step_id: 'draft', capability_id: 'compose', title: 'Draft the comparison', + arguments: {}, role: 'reason', estimated_cost: 'low', + }], + assumptions: [], + status: mode === 'auto' ? 'approved' : 'awaiting_approval', + approval: { mode, state: mode === 'auto' ? 'approved' : 'pending' }, + validation: { valid: true, errors: [], warnings: [], repairs: [] }, + }; +} + +/** A run-time notice, shaped as build_planning_thought serializes it. */ +function runNotice() { + return { + type: 'thought', message_id: null, step_index: 1, step_type: 'orchestration_planning', + content: NOTICE, + activity: { lane_key: 'orchestration', kind: 'orchestration_planning', title: 'Building a plan', status: 'running' }, + }; +} + +/** The run's terminal frame, shaped as build_run_done_event serializes it. */ +function runDone(plan, status = 'completed') { + return { + done: true, type: 'orchestration_done', conversation_id: plan.conversation_id, + message_id: `answer-${plan.conversation_id}`, run_id: plan.run_id, + full_content: status === 'completed' ? ANSWER : '', + hybrid_citations: [], web_search_citations: [], agent_citations: [], augmented: false, + generated_artifacts: [], orchestration: {}, status, outcome: status, turn_id: plan.turn_id, + attempt_index: 1, retry_of_run_id: null, failure: null, failures: [], recovery: null, + message_saved: status === 'completed', reasoning_adjustments: [], + }; +} + +/** Plan a turn through the real controller and return its plan once the stream has closed. */ +async function planTurn(streams, conversationId, mode = 'manual') { + const opened = streams.opened(PLAN_PATH); + const planning = startOrchestrationPlan({ + conversationId, message: 'Compare the quarterly reports.', approvalMode: mode, seeds: {}, + }); + const request = await opened; + assert.equal(chat().streaming, true); + assert.equal(chat().orchestrationSurface, 'planning', 'the planner takes the surface as planning'); + const plan = makePlan(conversationId, request.turn_id, mode); + streams.emit(PLAN_PATH, { type: 'orchestration_plan', plan, done: true }); + streams.end(PLAN_PATH); + return { plan, planning }; +} + +test('the planner takes the surface as planning and an approved run as running', async () => { + const conversation = 'surface-manual'; + openConversation(conversation); + const streams = holdStreams(); + try { + const { plan, planning } = await planTurn(streams, conversation); + await planning; + assert.equal(chat().streaming, false); + assert.equal(chat().orchestrationSurface, null, 'a plan awaiting approval holds no surface'); + assert.equal(runCardShowsProgress(conversation), false); + + const opened = streams.opened(RUN_PATH); + const running = approveAndRunPlan({ conversationId: conversation, turnId: plan.turn_id }); + await opened; + assert.equal(chat().streaming, true, 'Stop and the composer lock still follow the run'); + assert.equal(chat().orchestrationSurface, 'running'); + assert.equal(runCardShowsProgress(conversation), true, 'the card is drawing the run'); + + streams.emit(RUN_PATH, { type: 'orchestration_step', step_id: 'draft', status: 'running', summary: '' }); + streams.emit(RUN_PATH, runNotice()); + streams.emit(RUN_PATH, { type: 'orchestration_step', step_id: 'draft', status: 'completed', summary: '' }); + streams.emit(RUN_PATH, { content: ANSWER }); + streams.emit(RUN_PATH, runDone(plan)); + streams.end(RUN_PATH); + await running; + + assert.equal(chat().streaming, false); + assert.equal(chat().orchestrationSurface, null); + assert.equal(runCardShowsProgress(conversation), false); + const answer = chat().messages.find((message) => message.id === `answer-${conversation}`); + assert.equal(answer?.content, ANSWER); + // The bubble hid the live reasoning toggle during the run; the notice is not lost. + assert.deepEqual(answer.thoughts?.map((thought) => thought.content), [NOTICE]); + } finally { + streams.restore(); + } +}); + +test('an auto-approved plan hands the surface straight from planning to the run', async () => { + const conversation = 'surface-auto'; + openConversation(conversation); + const seen = []; + const unsubscribe = useChatStore.subscribe((state) => { + seen.push(`${state.streaming}:${state.orchestrationSurface}`); + }); + const streams = holdStreams(); + try { + const opened = streams.opened(RUN_PATH); + const { plan, planning } = await planTurn(streams, conversation, 'auto'); + await opened; + assert.equal(chat().orchestrationSurface, 'running'); + assert.equal(runCardShowsProgress(conversation), true); + + streams.emit(RUN_PATH, { content: ANSWER }); + streams.emit(RUN_PATH, runDone(plan)); + streams.end(RUN_PATH); + await planning; + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(chat().orchestrationSurface, null); + + const changes = seen.filter((entry, index) => entry !== seen[index - 1]); + assert.ok(!changes.includes('true:null'), `a chat-owned stream never appeared: ${changes.join(', ')}`); + const planningAt = changes.indexOf('true:planning'); + assert.ok(planningAt >= 0, 'the planner held the surface'); + assert.equal(changes[planningAt + 1], 'true:running', 'nothing came between the plan and its run'); + } finally { + unsubscribe(); + streams.restore(); + } +}); + +test('a chat reply sent while a run waits is not mistaken for the run', async () => { + const conversation = 'surface-waiting'; + openConversation(conversation); + const streams = holdStreams(); + try { + const { plan, planning } = await planTurn(streams, conversation); + await planning; + const opened = streams.opened(RUN_PATH); + void approveAndRunPlan({ conversationId: conversation, turnId: plan.turn_id }); + await opened; + streams.emit(RUN_PATH, runDone(plan, 'waiting')); + streams.end(RUN_PATH); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // A durable wait releases the surface but leaves the run in flight on its card. + assert.equal(chat().streaming, false); + assert.equal(chat().orchestrationSurface, null); + assert.equal(runCardShowsProgress(conversation), true); + + const chatOpened = streams.opened('/api/chat/stream'); + void chat().sendMessage('And what about costs?', {}); + await chatOpened; + assert.equal(chat().streaming, true); + assert.equal(chat().orchestrationSurface, null, 'the reply is a chat stream and shows Thinking'); + } finally { + streams.restore(); + } +}); + +test('a run keeps its phase when the reader leaves and comes back', () => { + const original = globalThis.fetch; + // Opening a conversation reads its messages; held forever, nothing else happens. + globalThis.fetch = () => new Promise(() => {}); + try { + openConversation('surface-elsewhere'); + chat().beginOrchestrationTurn('surface-away', '', false, undefined, undefined, 'running'); + assert.equal(chat().streaming, false, 'a turn out of sight does not touch this conversation'); + assert.equal(chat().orchestrationSurface, null); + + void chat().selectConversation('surface-away', { kind: 'personal' }); + assert.equal(chat().streaming, true); + assert.equal(chat().orchestrationSurface, 'running'); + + chat().startNewConversation(); + assert.equal(chat().streaming, false); + assert.equal(chat().orchestrationSurface, null); + + void chat().selectConversation('surface-away', { kind: 'personal' }); + assert.equal(chat().orchestrationSurface, 'running'); + + chat().settleOrchestrationTurn('surface-away', { status: 'planned' }); + assert.equal(chat().streaming, false); + assert.equal(chat().orchestrationSurface, null); + + void chat().selectConversation('surface-elsewhere', { kind: 'personal' }); + void chat().selectConversation('surface-away', { kind: 'personal' }); + assert.equal(chat().streaming, false, 'a settled turn does not come back'); + assert.equal(chat().orchestrationSurface, null); + } finally { + globalThis.fetch = original; + } +}); + +test('a turn re-keyed to the server conversation id keeps its phase', () => { + const original = globalThis.fetch; + globalThis.fetch = () => new Promise(() => {}); + try { + openConversation('surface-client-id'); + chat().beginOrchestrationTurn('surface-client-id', 'Compare the reports.', true, 'turn-1'); + assert.equal(chat().orchestrationSurface, 'planning', 'callers that predate the phase still plan'); + + chat().reassignOrchestrationTurn({ + fromConversationId: 'surface-client-id', toConversationId: 'surface-server-id', + fromTurnId: 'turn-1', toTurnId: 'turn-1', + }); + chat().startNewConversation(); + void chat().selectConversation('surface-server-id', { kind: 'personal' }); + assert.equal(chat().streaming, true); + assert.equal(chat().orchestrationSurface, 'planning'); + chat().settleOrchestrationTurn('surface-server-id', { status: 'planned' }); + } finally { + globalThis.fetch = original; + } +}); + +test('every write that takes the streaming flag says who owns it', () => { + const store = readFileSync(new URL('../application/v2_ui/src/stores/chatStore.ts', import.meta.url), 'utf8'); + const takes = store.match(/streaming: true,/g) ?? []; + const owners = [...store.matchAll(/streaming: true,\s*\n\s*orchestrationSurface: (null|phase),/g)] + .map((match) => match[1]); + assert.ok(takes.length >= 5, 'the chat starters and the orchestration seam are all present'); + assert.equal(owners.length, takes.length, 'a stream that takes `streaming` must name its owner'); + assert.deepEqual(owners.filter((owner) => owner === 'phase'), ['phase'], 'only the orchestration seam takes a phase'); + + const controller = readFileSync( + new URL('../application/v2_ui/src/lib/orchestrationController.ts', import.meta.url), 'utf8', + ); + const runs = controller.match(/beginOrchestrationTurn\([^)]*'running'\)/g) ?? []; + assert.deepEqual(runs, ["beginOrchestrationTurn(conversationId, '', false, undefined, undefined, 'running')"]); +}); diff --git a/functional_tests/test_v2_stream_reconnect.py b/functional_tests/test_v2_stream_reconnect.py index 46dde29e5..4dcd2e4f5 100644 --- a/functional_tests/test_v2_stream_reconnect.py +++ b/functional_tests/test_v2_stream_reconnect.py @@ -2,7 +2,7 @@ """ Functional test for V2 chat stream reconnection. -Version: 0.261.017 +Version: 0.261.256 Implemented in: 0.261.017 A chat answer is generated on the server and written into a stream session that outlives @@ -226,10 +226,15 @@ def test_reconnecting_and_reconnected_are_distinct_states(): "test exists to prevent" ) - # The label reverts once connected. - assert "connecting ? 'Reconnecting' : 'Thinking'" in message_list, ( - "The activity label must go back to Thinking once frames are arriving again" + # The label reverts once connected. Reconnecting is only ever a chat stream's state, and a chat + # stream's usual label is Thinking; an orchestration plan's is Planning. + assert "connecting ? 'Reconnecting' : activityLabel" in message_list, ( + "The activity label must go back to its usual wording once frames are arriving again" ) + assert ( + "const activityLabel = orchestrationSurface === 'planning' ? 'Planning' : 'Thinking';" + in message_list + ), "A chat stream's usual activity label must still be Thinking" print("Reconnect phase separation test passed!") return True diff --git a/ui_tests/test_v2_new_chat_reset.py b/ui_tests/test_v2_new_chat_reset.py index 1b6259370..131c820aa 100644 --- a/ui_tests/test_v2_new_chat_reset.py +++ b/ui_tests/test_v2_new_chat_reset.py @@ -1,7 +1,7 @@ # test_v2_new_chat_reset.py """ Browser regressions for starting a new chat in V2 while the open conversation is busy. -Version: 0.261.226 +Version: 0.261.256 Implemented in: 0.261.226 Clicking New chat while an orchestration turn was planning or running kept the old turn's @@ -11,6 +11,10 @@ placeholders in the new chat, and Home's Start chatting reopened whichever conversation was last open (issue #1617). +Since 0.261.256 a planning turn's bubble says "Planning" rather than "Thinking", and a running +plan shows its progress only on the plan card, so those are what the orchestration checks look +for. A chat reply still says "Thinking". + The production AppShell, Sidebar, HomePage, ChatPage, Composer, MessageList, stores, orchestration controller and SSE reader run in Chromium with production CSS. Only HTTP is deterministic: plan, run and chat streams are held open and fed one server-shaped frame at @@ -369,6 +373,16 @@ def thinking(page): return page.get_by_text("Thinking", exact=True) +def planning(page): + """The streaming bubble's label while an orchestration turn plans.""" + return page.get_by_text("Planning", exact=True) + + +def running_card(page): + """The plan card's running row, the only progress shown while a plan runs.""" + return page.get_by_role("button", name="Review the running plan") + + def stop_button(page): return page.get_by_role("button", name="Stop generating", exact=True) @@ -400,6 +414,8 @@ def expect_clean_new_chat(page, *absent): expect(page.get_by_role("heading", name="New chat", level=1)).to_be_visible() expect(page.get_by_text(EMPTY_STATE, exact=True)).to_be_visible() expect(thinking(page)).to_have_count(0) + expect(planning(page)).to_have_count(0) + expect(running_card(page)).to_have_count(0) expect(stop_button(page)).to_have_count(0) expect(send_button(page)).to_be_visible() for text in absent: @@ -446,7 +462,7 @@ def test_new_chat_while_planning_starts_clean_and_returning_restores_the_turn(ch mount(page, api) turn = start_plan(page) persist_question(api, turn) - expect(thinking(page)).to_be_visible() + expect(planning(page)).to_be_visible() expect(stop_button(page)).to_be_visible() expect(page.get_by_text(QUESTION, exact=True)).to_be_visible() @@ -457,7 +473,7 @@ def test_new_chat_while_planning_starts_clean_and_returning_restores_the_turn(ch # working, with Stop, and its next reasoning step lands there as usual. open_from_rail(page, BUSY_TITLE) expect(page.get_by_text(QUESTION, exact=True)).to_be_visible() - expect(thinking(page)).to_be_visible() + expect(planning(page)).to_be_visible() expect(stop_button(page)).to_be_visible() emit(page, PLAN, BUSY, planning_thought(DECIDING)) expect(page.get_by_role("button", name="1 reasoning step", exact=True)).to_be_visible() @@ -469,9 +485,10 @@ def test_new_chat_while_planning_starts_clean_and_returning_restores_the_turn(ch expect(approve_button(page)).to_have_count(0) expect_clean_new_chat(page, QUESTION, EARLIER_ANSWER) - # Its turn settled while the reader was away, so coming back shows the plan, not Thinking. + # Its turn settled while the reader was away, so coming back shows the plan, not Planning. open_from_rail(page, BUSY_TITLE) expect(approve_button(page).first).to_be_visible() + expect(planning(page)).to_have_count(0) expect(thinking(page)).to_have_count(0) expect(stop_button(page)).to_have_count(0) expect(send_button(page)).to_be_visible() @@ -485,7 +502,9 @@ def test_new_chat_while_a_plan_runs_keeps_the_answer_in_its_own_chat_and_can_sen plan = finish_plan(page, turn) approve_button(page).first.click() wait_for_stream(page, RUN, BUSY) - expect(thinking(page)).to_be_visible() + # A run's progress is on its plan card alone; Stop still belongs to it. + expect(running_card(page)).to_be_visible() + expect(thinking(page)).to_have_count(0) expect(stop_button(page)).to_be_visible() click_new_chat(page) @@ -507,14 +526,14 @@ def test_new_chat_while_a_plan_runs_keeps_the_answer_in_its_own_chat_and_can_sen send_button(page).click() wait_for_stream(page, PLAN, CREATED) expect(page.get_by_text(FOLLOW_UP, exact=True)).to_be_visible() - expect(thinking(page)).to_be_visible() + expect(planning(page)).to_be_visible() assert chat_state(page)["active"] == CREATED assert len(api.created()) == 1 expect(page.get_by_text(QUESTION, exact=True)).to_have_count(0) # Stop now belongs to the new chat's turn, and ends it. stop_button(page).click() - expect(thinking(page)).to_have_count(0) + expect(planning(page)).to_have_count(0) expect(send_button(page)).to_be_visible() diff --git a/ui_tests/test_v2_orchestration_streaming_bubble.py b/ui_tests/test_v2_orchestration_streaming_bubble.py index 809e21dea..68d909b1d 100644 --- a/ui_tests/test_v2_orchestration_streaming_bubble.py +++ b/ui_tests/test_v2_orchestration_streaming_bubble.py @@ -1,20 +1,27 @@ # test_v2_orchestration_streaming_bubble.py """ Browser regression for the V2 orchestration streaming bubble. -Version: 0.261.204 +Version: 0.261.256 Implemented in: 0.261.204 -While an orchestrated turn is planned or run, the streaming bubble shows only the -"N reasoning steps" toggle and the "Thinking" indicator. It used to also draw an -Orchestration progress card (heading, "Current step: Building a plan", a percentage, -a step count and a progress bar) that repeated what the toggle and the plan card -already say, so that card is no longer drawn, live or in a finished answer's -expanded reasoning. Tabular analysis keeps its progress card. +An orchestrated turn shows one progress indicator at a time. -The production controller, stores, SSE reader, MessageList and plan card run in -Chromium with production CSS. Only HTTP is deterministic: the plan and run streams -are held open and fed one server-shaped frame at a time, so each intermediate state -of the bubble can be inspected. No live model, deployment or workspace content is used. +While a turn plans, the streaming bubble shows only the "N reasoning steps" toggle and a +"Planning" indicator. It used to also draw an Orchestration progress card (heading, "Current +step: Building a plan", a percentage, a step count and a progress bar) that repeated what the +toggle and the plan card already say (0.261.204), and it said "Thinking" (0.261.256). + +While an approved plan runs, the plan card is the only indicator. The streaming bubble used to +show "Thinking", and its run notices, beside the card's own progress line (0.261.256). The card's +status line now says what the run is doing: "Starting", the running step's kind of work and title, +and "Preparing the answer" once every step has settled. The run's notices stay with the finished +answer's reasoning steps. Any stream that is not the run's own, such as a chat reply sent while a +run waits, still shows "Thinking". Tabular analysis keeps its progress card. + +The production controller, stores, SSE reader, MessageList and plan card run in Chromium with +production CSS. Only HTTP is deterministic: the plan and run streams are held open and fed one +server-shaped frame at a time, so each intermediate state of the bubble can be inspected. No live +model, deployment or workspace content is used. Build CSS with the existing V2 build, keeping outputs in UI test artifacts: npm --prefix .\\application\\v2_ui run build -- --outDir ..\\..\\ui_tests\\artifacts\\orchestration-plan-editor @@ -79,6 +86,39 @@ } """ +# Records every activity label that reaches the DOM, including one drawn for a single frame and +# replaced before a locator could see it, so a test can prove a label never appeared at all. +WATCH_ACTIVITY_LABELS = r""" +() => { + window.__activityLabels = []; + const record = (text) => { + const value = (text || '').trim(); + if (value === 'Thinking' || value === 'Planning') { + window.__activityLabels.push(value); + } + }; + const visit = (node) => { + if (node.nodeType === Node.TEXT_NODE) { + record(node.data); + return; + } + const walker = document.createTreeWalker(node, NodeFilter.SHOW_TEXT); + while (walker.nextNode()) { + record(walker.currentNode.data); + } + }; + visit(document.body); + new MutationObserver((mutations) => { + for (const mutation of mutations) { + if (mutation.type === 'characterData') { + record(mutation.target.data); + } + mutation.addedNodes.forEach(visit); + } + }).observe(document.body, { childList: true, subtree: true, characterData: true }); +} +""" + def planning_thought(content, status="running"): """A planner reasoning step, shaped exactly as build_planning_thought serializes it.""" @@ -97,6 +137,11 @@ def planning_thought(content, status="running"): } +def step_frame(step_id, status): + """A plan step changing state, shaped as build_step_event serializes it.""" + return {"type": "orchestration_step", "step_id": step_id, "status": status, "summary": ""} + + def run_done(plan, turn): """The run's terminal frame, shaped as build_run_done_event serializes a completed run.""" return { @@ -220,10 +265,34 @@ def expect_toggle_first(button): assert button.evaluate("(node) => node.parentElement.firstElementChild === node") -def start_plan(page): +def watch_activity_labels(page): + page.evaluate(WATCH_ACTIVITY_LABELS) + + +def activity_labels(page): + """Every Thinking or Planning label drawn since watch_activity_labels, in order.""" + return page.evaluate("() => window.__activityLabels") + + +def expect_run_status(page, text): + """The running plan card's status line, which is the run's only progress indicator.""" + expect(page.get_by_text(text, exact=True)).to_have_attribute("role", "status") + + +def expect_only_the_run_card(page): + """While a plan runs, the card is drawn and the streaming bubble is not.""" + expect(page.get_by_role("button", name="Review the running plan")).to_be_visible() + expect(page.get_by_text("Thinking", exact=True)).to_have_count(0) + expect(page.get_by_text("Planning", exact=True)).to_have_count(0) + # The bubble's animated dots, and its live reasoning toggle, are the bubble. + expect(page.locator(".animate-bounce")).to_have_count(0) + expect(page.get_by_role("button", name=re.compile(r"^\d+ reasoning steps?$"))).to_have_count(0) + + +def start_plan(page, approval_mode="manual"): page.evaluate( """(spec) => { void window.OrchHarness.controller.startOrchestrationPlan(spec); }""", - {"conversationId": CONVERSATION, "message": QUESTION, "approvalMode": "manual", "seeds": {}}, + {"conversationId": CONVERSATION, "message": QUESTION, "approvalMode": approval_mode, "seeds": {}}, ) wait_for_stream(page, PLAN) return page.evaluate( @@ -242,25 +311,26 @@ def finish_plan(page, turn): return plan -def test_planning_bubble_shows_reasoning_toggle_and_thinking_without_a_progress_card(bubble_ui): +def test_planning_bubble_shows_reasoning_toggle_and_planning_without_a_progress_card(bubble_ui): page, api = bubble_ui turn = start_plan(page) - thinking = page.get_by_text("Thinking", exact=True) - expect(thinking).to_be_visible() + planning = page.get_by_text("Planning", exact=True) + expect(planning).to_be_visible() + expect(page.get_by_text("Thinking", exact=True)).to_have_count(0) expect(page.get_by_text(QUESTION, exact=True)).to_be_visible() emit(page, PLAN, planning_thought(DECIDING)) steps = toggle(page, 1) expect(steps).to_be_visible() expect(steps).to_have_attribute("aria-expanded", "false") - expect(thinking).to_be_visible() + expect(planning).to_be_visible() expect_no_progress_card(page) # The card repeated the latest planner sentence under its bar; collapsed, it stays hidden. expect(page.get_by_text(DECIDING, exact=True)).to_have_count(0) expect_toggle_first(steps) toggle_box = steps.bounding_box() - thinking_box = thinking.bounding_box() - assert toggle_box and thinking_box and toggle_box["y"] < thinking_box["y"] + planning_box = planning.bounding_box() + assert toggle_box and planning_box and toggle_box["y"] < planning_box["y"] steps.click() expect(steps).to_have_attribute("aria-expanded", "true") @@ -269,13 +339,14 @@ def test_planning_bubble_shows_reasoning_toggle_and_thinking_without_a_progress_ expect_no_progress_card(page) finish_plan(page, turn) - expect(thinking).to_have_count(0) + expect(planning).to_have_count(0) expect(page.get_by_text(DECIDING, exact=True)).to_have_count(0) expect_no_progress_card(page) -def test_run_and_finished_answer_show_no_orchestration_progress_card(bubble_ui): +def test_a_run_shows_only_the_plan_card_and_its_answer_keeps_the_notice(bubble_ui): page, api = bubble_ui + watch_activity_labels(page) turn = start_plan(page) emit(page, PLAN, planning_thought(DECIDING)) expect(toggle(page, 1)).to_be_visible() @@ -283,24 +354,36 @@ def test_run_and_finished_answer_show_no_orchestration_progress_card(bubble_ui): page.get_by_role("button", name="Approve and run the plan").first.click() wait_for_stream(page, RUN) - thinking = page.get_by_text("Thinking", exact=True) - expect(thinking).to_be_visible() - # Run progress stays on the plan card, which keeps its own compact line. - expect(page.get_by_role("button", name="Review the running plan")).to_be_visible() + expect_only_the_run_card(page) + expect_run_status(page, "Starting") + + emit(page, RUN, step_frame("read", "running")) + expect_run_status(page, "Gathering: Read quarterly reports") + # A run notice is a reasoning step. It no longer opens a live toggle beside the card. emit(page, RUN, planning_thought(MEMORY_NOTICE)) - steps = toggle(page, 1) - expect(steps).to_be_visible() - expect(thinking).to_be_visible() expect(page.get_by_text(MEMORY_NOTICE, exact=True)).to_have_count(0) - expect_toggle_first(steps) + expect_only_the_run_card(page) expect_no_progress_card(page) + emit(page, RUN, step_frame("read", "completed")) + emit(page, RUN, step_frame("research", "running")) + expect_run_status(page, "Gathering: Investigate context") + emit(page, RUN, step_frame("research", "completed")) + emit(page, RUN, step_frame("answer", "running")) + expect_run_status(page, "Reasoning: Write the comparison") + expect(page.get_by_text("2/3", exact=True)).to_be_visible() + + # The answer arrives in one piece after every step settles; the wait before it is named. + emit(page, RUN, step_frame("answer", "completed")) + expect_run_status(page, "Preparing the answer") + expect_only_the_run_card(page) + emit(page, RUN, {"content": ANSWER}) emit(page, RUN, run_done(plan, turn)) page.evaluate("(path) => window.endStream(path)", RUN) expect(page.get_by_text(ANSWER, exact=True)).to_be_visible() - expect(thinking).to_have_count(0) + expect(page.get_by_role("button", name="Review the running plan")).to_have_count(0) saved = toggle(page, 1) expect(saved).to_be_visible() @@ -309,6 +392,60 @@ def test_run_and_finished_answer_show_no_orchestration_progress_card(bubble_ui): expect(page.get_by_text(MEMORY_NOTICE, exact=True)).to_be_visible() expect_no_progress_card(page) + labels = activity_labels(page) + assert "Planning" in labels, labels + assert "Thinking" not in labels, f"Thinking was drawn during the orchestrated turn: {labels}" + + +def test_an_auto_approved_plan_goes_from_planning_straight_to_the_run_card(bubble_ui): + page, api = bubble_ui + watch_activity_labels(page) + turn = start_plan(page, approval_mode="auto") + expect(page.get_by_text("Planning", exact=True)).to_be_visible() + + plan = editor_tests.make_plan(CONVERSATION, turn, mode="auto") + plan["approval"] = {"mode": "auto", "timeout_seconds": 0, "state": "approved"} + plan["status"] = "approved" + emit(page, PLAN, {"type": "orchestration_plan", "plan": plan, "done": True}) + page.evaluate("(path) => window.endStream(path)", PLAN) + wait_for_stream(page, RUN) + expect_only_the_run_card(page) + expect_run_status(page, "Starting") + + emit(page, RUN, {"content": ANSWER}) + emit(page, RUN, run_done(plan, turn)) + page.evaluate("(path) => window.endStream(path)", RUN) + expect(page.get_by_text(ANSWER, exact=True)).to_be_visible() + expect(page.get_by_role("button", name="Review the running plan")).to_have_count(0) + + labels = activity_labels(page) + assert "Planning" in labels, labels + assert "Thinking" not in labels, f"Thinking flashed between the plan and its run: {labels}" + + +def test_a_stream_that_is_not_the_run_still_shows_thinking_beside_its_card(bubble_ui): + page, api = bubble_ui + turn = start_plan(page) + plan = finish_plan(page, turn) + page.get_by_role("button", name="Approve and run the plan").first.click() + wait_for_stream(page, RUN) + review = page.get_by_role("button", name="Review the running plan") + thinking = page.get_by_text("Thinking", exact=True) + expect_only_the_run_card(page) + + # A chat stream takes the streaming flag with no orchestration owner. That is the state a reply + # sent while a run waits in the same conversation leaves, with the run still on its card. Its + # Thinking must not be mistaken for the run's and hidden. + page.evaluate("() => window.OrchHarness.stores.chat.useChatStore.setState({orchestrationSurface: null})") + expect(thinking).to_be_visible() + expect(review).to_be_visible() + + emit(page, RUN, {"content": ANSWER}) + emit(page, RUN, run_done(plan, turn)) + page.evaluate("(path) => window.endStream(path)", RUN) + expect(page.get_by_text(ANSWER, exact=True)).to_be_visible() + expect(thinking).to_have_count(0) + def test_tabular_streaming_bubble_keeps_its_progress_card(bubble_ui): page, api = bubble_ui diff --git a/ui_tests/test_v2_reasoning_controls.py b/ui_tests/test_v2_reasoning_controls.py index be2c973cd..703397b7f 100644 --- a/ui_tests/test_v2_reasoning_controls.py +++ b/ui_tests/test_v2_reasoning_controls.py @@ -1,7 +1,7 @@ # test_v2_reasoning_controls.py """ Real-Composer reasoning, capability selections, and saved-plan notice regressions. -Version: 0.261.132 +Version: 0.261.256 Implemented in: 0.261.104; Image requests orchestration image proposals since 0.261.132 Reuse the local/Azure Playwright fixtures without live model, Azure or retrieval calls. @@ -545,7 +545,8 @@ def test_run_thought_corrections_appear_before_completion_and_latest_stage_wins( }], } page.evaluate("(event) => window.emitReasoningRunEvent(event)", correction) - notice = page.locator("#mount-b").get_by_role("status") + # Filtered to the notice: the running card's own status line ("Starting") is a status as well. + notice = page.locator("#mount-b").get_by_role("status").filter(has_text="could not be used") expect(notice).to_contain_text("Planner: Minimal could not be used for gpt-5.6-luna; using Model default.") expect(notice).not_to_contain_text("using Low") assert page.evaluate("() => window.OrchHarness.stores.chat.useChatStore.getState().streaming")