Skip to content
This repository was archived by the owner on May 20, 2026. It is now read-only.

Resolve endpoint once per tool-calling round instead of per-method #5031

Open
abadawi591 wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Resolve endpoint once per tool-calling round instead of per-method #5031

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c969379
Resolve endpoint once per tool-calling round instead of per-method
abadawi591 Apr 7, 2026
87a691b
Fix literal \\n in import statements
abadawi591 Apr 7, 2026
e806885
Update test subclasses to match new abstract method signatures
abadawi591 Apr 7, 2026
e7f742d
Override resolveEndpoint in DefaultToolCallingLoop
abadawi591 Apr 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 16 additions & 10 deletions src/extension/intents/node/toolCallingLoop.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { IFileSystemService } from '../../../platform/filesystem/common/fileSyst
import { IGitService } from '../../../platform/git/common/gitService';
import { ILogService } from '../../../platform/log/common/logService';
import { isOpenAIContextManagementResponse, OpenAiFunctionDef } from '../../../platform/networking/common/fetch';
import { IMakeChatRequestOptions } from '../../../platform/networking/common/networking';
import { IChatEndpoint, IMakeChatRequestOptions } from '../../../platform/networking/common/networking';
import { OpenAIContextManagementResponse } from '../../../platform/networking/common/openai';
import { CopilotChatAttr, emitAgentTurnEvent, emitSessionStartEvent, GenAiAttr, GenAiMetrics, GenAiOperationName, GenAiProviderName, resolveWorkspaceOTelMetadata, StdAttr, truncateForOTel, workspaceMetadataToOTelAttributes } from '../../../platform/otel/common/index';
import { IOTelService, ISpanHandle, SpanKind, SpanStatusCode } from '../../../platform/otel/common/otelService';
Expand Down Expand Up @@ -213,11 +213,16 @@ export abstract class ToolCallingLoop<TOptions extends IToolCallingLoopOptions =
super();
}

/** Resolves the endpoint to use for this round. Override in subclasses that need a custom endpoint. */
protected async resolveEndpoint(): Promise<IChatEndpoint> {
return this._endpointProvider.getChatEndpoint(this.options.request);
}

/** Builds a prompt with the context. */
protected abstract buildPrompt(buildPromptContext: IBuildPromptContext, progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart>, token: CancellationToken): Promise<IBuildPromptResult>;
protected abstract buildPrompt(endpoint: IChatEndpoint, buildPromptContext: IBuildPromptContext, progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart>, token: CancellationToken): Promise<IBuildPromptResult>;

/** Gets the tools that should be callable by the model. */
protected abstract getAvailableTools(outputStream: ChatResponseStream | undefined, token: CancellationToken): Promise<LanguageModelToolInformation[]>;
protected abstract getAvailableTools(endpoint: IChatEndpoint, outputStream: ChatResponseStream | undefined, token: CancellationToken): Promise<LanguageModelToolInformation[]>;

/** Creates the prompt context for the request. */
protected createPromptContext(availableTools: LanguageModelToolInformation[], outputStream: ChatResponseStream | undefined): Mutable<IBuildPromptContext> {
Expand Down Expand Up @@ -266,6 +271,7 @@ export abstract class ToolCallingLoop<TOptions extends IToolCallingLoopOptions =
}

protected abstract fetch(
endpoint: IChatEndpoint,
options: ToolCallingLoopFetchOptions,
token: CancellationToken
): Promise<ChatResponse>;
Comment thread
abadawi591 marked this conversation as resolved.
Expand Down Expand Up @@ -1254,7 +1260,8 @@ export abstract class ToolCallingLoop<TOptions extends IToolCallingLoopOptions =

/** Runs a single iteration of the tool calling loop. */
public async runOne(outputStream: ChatResponseStream | undefined, iterationNumber: number, token: CancellationToken): Promise<IToolCallSingleResult> {
let availableTools = await this.getAvailableTools(outputStream, token);
const endpoint = await this.resolveEndpoint();
let availableTools = await this.getAvailableTools(endpoint, outputStream, token);

// Emit tools_available on the agent span once, before the first CHAT span
// starts in fetch(). This lets the debug logger write tools_*.json early.
Expand All @@ -1271,14 +1278,14 @@ export abstract class ToolCallingLoop<TOptions extends IToolCallingLoopOptions =
markChatExt(this.options.conversation.sessionId, ChatExtPerfMark.WillBuildPrompt);
let buildPromptResult: IBuildPromptResult;
try {
buildPromptResult = await this.buildPrompt2(context, outputStream, token);
buildPromptResult = await this.buildPrompt2(endpoint, context, outputStream, token);
} finally {
markChatExt(this.options.conversation.sessionId, ChatExtPerfMark.DidBuildPrompt);
}
this.throwIfCancelled(token);
this.turn.addReferences(buildPromptResult.references);
// Possible the tool call resulted in new tools getting added.
availableTools = await this.getAvailableTools(outputStream, token);
availableTools = await this.getAvailableTools(endpoint, outputStream, token);

// Apply debug prompt/tool overrides from YAML file, only when the setting is explicitly configured
const promptOverrideFile = this._configurationService.getConfig(ConfigKey.Advanced.DebugPromptOverrideFile);
Expand Down Expand Up @@ -1317,7 +1324,6 @@ export abstract class ToolCallingLoop<TOptions extends IToolCallingLoopOptions =
});
}

const endpoint = await this._endpointProvider.getChatEndpoint(this.options.request);
const tokenizer = endpoint.acquireTokenizer();
const promptTokenLength = await tokenizer.countMessagesTokens(effectiveBuildPromptResult.messages);
const toolTokenCount = availableTools.length > 0 ? await tokenizer.countToolTokens(availableTools) : 0;
Expand Down Expand Up @@ -1403,7 +1409,7 @@ export abstract class ToolCallingLoop<TOptions extends IToolCallingLoopOptions =
let compaction: OpenAIContextManagementResponse | undefined;
this._isInlineSummarizationRequest = inlineSummarizationRequested;
markChatExt(this.options.conversation.sessionId, ChatExtPerfMark.WillFetch);
const fetchResult = await this.fetch({
const fetchResult = await this.fetch(endpoint, {
messages: this.applyMessagePostProcessing(effectiveBuildPromptResult.messages, { stripOrphanedToolCalls: isGeminiFamily(endpoint) }),
turnId: this.turn.id,
finishedCb: async (text, index, delta) => {
Expand Down Expand Up @@ -1791,14 +1797,14 @@ export abstract class ToolCallingLoop<TOptions extends IToolCallingLoopOptions =
return filtered;
}

private async buildPrompt2(buildPromptContext: IBuildPromptContext, stream: ChatResponseStream | undefined, token: CancellationToken): Promise<IBuildPromptResult> {
private async buildPrompt2(endpoint: IChatEndpoint, buildPromptContext: IBuildPromptContext, stream: ChatResponseStream | undefined, token: CancellationToken): Promise<IBuildPromptResult> {
const progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart> = {
report(obj) {
stream?.push(obj);
}
};

const buildPromptResult = await this.buildPrompt(buildPromptContext, progress, token);
const buildPromptResult = await this.buildPrompt(endpoint, buildPromptContext, progress, token);
for (const metadata of buildPromptResult.metadata.getAll(ToolResultMetadata)) {
this.logToolResult(buildPromptContext, metadata);
this.toolCallResults[metadata.toolCallId] = metadata.result;
Expand Down
12 changes: 5 additions & 7 deletions src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import { IRequestLogger } from '../../../platform/requestLogger/node/requestLogger';
import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService';
import { ITelemetryService } from '../../../platform/telemetry/common/telemetry';
import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation';
import { IChatEndpoint } from '../../../platform/networking/common/networking';\nimport { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation';

Check failure on line 20 in src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx

View workflow job for this annotation

GitHub Actions / Test (Linux) 

Unexpected keyword or identifier.

Check failure on line 20 in src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx

View workflow job for this annotation

GitHub Actions / Test (Linux) 

Unknown keyword or identifier. Did you mean 'import'?

Check failure on line 20 in src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx

View workflow job for this annotation

GitHub Actions / Test (Linux) 

Invalid character.

Check failure on line 20 in src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx

View workflow job for this annotation

GitHub Actions / Test (Windows) 

Unexpected keyword or identifier.

Check failure on line 20 in src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx

View workflow job for this annotation

GitHub Actions / Test (Windows) 

Unknown keyword or identifier. Did you mean 'import'?

Check failure on line 20 in src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx

View workflow job for this annotation

GitHub Actions / Test (Windows) 

Invalid character.
Comment thread
abadawi591 marked this conversation as resolved.
Outdated
import { ChatResponseProgressPart, ChatResponseReferencePart } from '../../../vscodeTypes';
import { IToolCallingLoopOptions, ToolCallingLoop, ToolCallingLoopFetchOptions } from '../../intents/node/toolCallingLoop';
import { IBuildPromptContext } from '../../prompt/common/intents';
Expand Down Expand Up @@ -52,12 +52,11 @@
super(options, instantiationService, endpointProvider, logService, requestLogger, authenticationChatUpgradeService, telemetryService, configurationService, experimentationService, chatHookService, sessionTranscriptService, fileSystemService, otelService, gitService);
}

private async getEndpoint() {
protected override async resolveEndpoint(): Promise<IChatEndpoint> {
return await this.endpointProvider.getChatEndpoint('copilot-fast');
}

protected async buildPrompt(buildPromptContext: IBuildPromptContext, progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart>, token: CancellationToken): Promise<IBuildPromptResult> {
const endpoint = await this.getEndpoint();
protected async buildPrompt(endpoint: IChatEndpoint, buildPromptContext: IBuildPromptContext, progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart>, token: CancellationToken): Promise<IBuildPromptResult> {
const renderer = PromptRenderer.create(
this.instantiationService,
endpoint,
Expand All @@ -70,7 +69,7 @@
return await renderer.render(progress, token);
}

protected async getAvailableTools(): Promise<LanguageModelToolInformation[]> {
protected async getAvailableTools(_endpoint: IChatEndpoint): Promise<LanguageModelToolInformation[]> {
if (this.options.conversation.turns.length > 5) {
return []; // force a response
}
Expand All @@ -90,8 +89,7 @@
}];
}

protected async fetch(opts: ToolCallingLoopFetchOptions, token: CancellationToken): Promise<ChatResponse> {
const endpoint = await this.getEndpoint();
protected async fetch(endpoint: IChatEndpoint, opts: ToolCallingLoopFetchOptions, token: CancellationToken): Promise<ChatResponse> {
return endpoint.makeChatRequest2({
...opts,
debugName: McpToolCallingLoop.ID,
Expand Down
12 changes: 5 additions & 7 deletions src/extension/prompt/node/codebaseToolCalling.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { IOTelService } from '../../../platform/otel/common/otelService';
import { IRequestLogger } from '../../../platform/requestLogger/node/requestLogger';
import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService';
import { ITelemetryService } from '../../../platform/telemetry/common/telemetry';
import { IChatEndpoint } from '../../../platform/networking/common/networking';
import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation';
import { ChatResponseProgressPart, ChatResponseReferencePart } from '../../../vscodeTypes';
import { IToolCallingLoopOptions, ToolCallingLoop, ToolCallingLoopFetchOptions } from '../../intents/node/toolCallingLoop';
Expand Down Expand Up @@ -56,16 +57,15 @@ export class CodebaseToolCallingLoop extends ToolCallingLoop<ICodebaseToolCallin
super(options, instantiationService, endpointProvider, logService, requestLogger, authenticationChatUpgradeService, telemetryService, configurationService, experimentationService, chatHookService, sessionTranscriptService, fileSystemService, otelService, gitService);
}

private async getEndpoint(request: ChatRequest) {
protected override async resolveEndpoint(): Promise<IChatEndpoint> {
let endpoint = await this.endpointProvider.getChatEndpoint(this.options.request);
if (!endpoint.supportsToolCalls) {
endpoint = await this.endpointProvider.getChatEndpoint('copilot-base');
}
return endpoint;
}

protected async buildPrompt(buildPromptContext: IBuildPromptContext, progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart>, token: CancellationToken): Promise<IBuildPromptResult> {
const endpoint = await this.getEndpoint(this.options.request);
protected async buildPrompt(endpoint: IChatEndpoint, buildPromptContext: IBuildPromptContext, progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart>, token: CancellationToken): Promise<IBuildPromptResult> {
const renderer = PromptRenderer.create(
this.instantiationService,
endpoint,
Expand All @@ -77,13 +77,11 @@ export class CodebaseToolCallingLoop extends ToolCallingLoop<ICodebaseToolCallin
return await renderer.render(progress, token);
}

protected async getAvailableTools(): Promise<LanguageModelToolInformation[]> {
const endpoint = await this.getEndpoint(this.options.request);
protected async getAvailableTools(endpoint: IChatEndpoint): Promise<LanguageModelToolInformation[]> {
return this.toolsService.getEnabledTools(this.options.request, endpoint, tool => tool.tags.includes('vscode_codesearch'));
}

protected async fetch({ messages, finishedCb, requestOptions }: ToolCallingLoopFetchOptions, token: CancellationToken): Promise<ChatResponse> {
const endpoint = await this.getEndpoint(this.options.request);
protected async fetch(endpoint: IChatEndpoint, { messages, finishedCb, requestOptions }: ToolCallingLoopFetchOptions, token: CancellationToken): Promise<ChatResponse> {
return endpoint.makeChatRequest(
CodebaseToolCallingLoop.ID,
messages,
Expand Down
7 changes: 4 additions & 3 deletions src/extension/prompt/node/defaultIntentRequestHandler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import { IOctoKitService } from '../../../platform/github/common/githubService';
import { HAS_IGNORED_FILES_MESSAGE } from '../../../platform/ignore/common/ignoreService';
import { ILogService } from '../../../platform/log/common/logService';
import { isAnthropicToolSearchEnabled } from '../../../platform/networking/common/anthropic';
import { IChatEndpoint } from '../../../platform/networking/common/networking';
import { FilterReason } from '../../../platform/networking/common/openai';
import { IOTelService } from '../../../platform/otel/common/otelService';
import { CapturingToken } from '../../../platform/requestLogger/common/capturingToken';
Expand Down Expand Up @@ -679,13 +680,13 @@ class DefaultToolCallingLoop extends ToolCallingLoop<IDefaultToolLoopOptions> {
}
}

protected override async buildPrompt(buildPromptContext: IBuildPromptContext, progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart>, token: CancellationToken): Promise<IBuildPromptResult> {
protected override async buildPrompt(_endpoint: IChatEndpoint, buildPromptContext: IBuildPromptContext, progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart>, token: CancellationToken): Promise<IBuildPromptResult> {
const buildPromptResult = await this.options.invocation.buildPrompt(buildPromptContext, progress, token);
this.fixMessageNames(buildPromptResult.messages);
return buildPromptResult;
}

protected override async fetch(opts: ToolCallingLoopFetchOptions, token: CancellationToken): Promise<ChatResponse> {
protected override async fetch(_endpoint: IChatEndpoint, opts: ToolCallingLoopFetchOptions, token: CancellationToken): Promise<ChatResponse> {
const messageSourcePrefix = this.options.location === ChatLocation.Editor ? 'inline' : 'chat';
const baseDebugName = this.options.request.subAgentInvocationId ?
`tool/runSubagent${this.options.request.subAgentName ? `-${this.options.request.subAgentName}` : ''}` :
Expand Down Expand Up @@ -729,7 +730,7 @@ class DefaultToolCallingLoop extends ToolCallingLoop<IDefaultToolLoopOptions> {
}, token);
}

protected override async getAvailableTools(outputStream: ChatResponseStream | undefined, token: CancellationToken): Promise<LanguageModelToolInformation[]> {
protected override async getAvailableTools(_endpoint: IChatEndpoint, outputStream: ChatResponseStream | undefined, token: CancellationToken): Promise<LanguageModelToolInformation[]> {
const tools = await this.options.invocation.getAvailableTools?.() ?? [];
Comment thread
abadawi591 marked this conversation as resolved.

// Skip tool grouping when Anthropic tool search is enabled
Expand Down
12 changes: 5 additions & 7 deletions src/extension/prompt/node/executionSubagentToolCallingLoop.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { IOTelService } from '../../../platform/otel/common/otelService';
import { IRequestLogger } from '../../../platform/requestLogger/node/requestLogger';
import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService';
import { ITelemetryService } from '../../../platform/telemetry/common/telemetry';
import { IChatEndpoint } from '../../../platform/networking/common/networking';
import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation';
import { ChatResponseProgressPart, ChatResponseReferencePart } from '../../../vscodeTypes';
import { IToolCallingLoopOptions, ToolCallingLoop, ToolCallingLoopFetchOptions } from '../../intents/node/toolCallingLoop';
Expand Down Expand Up @@ -77,7 +78,7 @@ export class ExecutionSubagentToolCallingLoop extends ToolCallingLoop<IExecution
/**
* Get the endpoint to use for the execution subagent
*/
private async getEndpoint() {
protected override async resolveEndpoint(): Promise<IChatEndpoint> {
const modelName = this._configurationService.getConfig(ConfigKey.Advanced.ExecutionSubagentModel) as ChatEndpointFamily;
if (modelName) {
try {
Expand All @@ -97,8 +98,7 @@ export class ExecutionSubagentToolCallingLoop extends ToolCallingLoop<IExecution
}
}

protected async buildPrompt(buildpromptContext: IBuildPromptContext, progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart>, token: CancellationToken): Promise<IBuildPromptResult> {
const endpoint = await this.getEndpoint();
protected async buildPrompt(endpoint: IChatEndpoint, buildpromptContext: IBuildPromptContext, progress: Progress<ChatResponseReferencePart | ChatResponseProgressPart>, token: CancellationToken): Promise<IBuildPromptResult> {
const maxExecutionTurns = this._configurationService.getExperimentBasedConfig(ConfigKey.Advanced.ExecutionSubagentToolCallLimit, this._experimentationService);
const renderer = PromptRenderer.create(
this.instantiationService,
Expand All @@ -112,8 +112,7 @@ export class ExecutionSubagentToolCallingLoop extends ToolCallingLoop<IExecution
return await renderer.render(progress, token);
}

protected async getAvailableTools(): Promise<LanguageModelToolInformation[]> {
const endpoint = await this.getEndpoint();
protected async getAvailableTools(endpoint: IChatEndpoint): Promise<LanguageModelToolInformation[]> {
const allTools = this.toolsService.getEnabledTools(this.options.request, endpoint);

const allowedExecutionTools = new Set([
Expand All @@ -123,8 +122,7 @@ export class ExecutionSubagentToolCallingLoop extends ToolCallingLoop<IExecution
return allTools.filter(tool => allowedExecutionTools.has(tool.name as ToolName));
}

protected async fetch({ messages, finishedCb, requestOptions, enableThinking, reasoningEffort }: ToolCallingLoopFetchOptions, token: CancellationToken): Promise<ChatResponse> {
const endpoint = await this.getEndpoint();
protected async fetch(endpoint: IChatEndpoint, { messages, finishedCb, requestOptions, enableThinking, reasoningEffort }: ToolCallingLoopFetchOptions, token: CancellationToken): Promise<ChatResponse> {
return endpoint.makeChatRequest2({
debugName: ExecutionSubagentToolCallingLoop.ID,
messages,
Expand Down
Loading
Loading