From 6809e437fbe015334a435c95188ba906a7400881 Mon Sep 17 00:00:00 2001 From: vmelikyan Date: Sun, 15 Feb 2026 00:24:30 -0800 Subject: [PATCH] Improve system prompts cleanup agent pricing config add session cost --- src/app/api/v2/ai/chat/[buildUuid]/route.ts | 14 +- src/components/ai-agent/ChatContainer.tsx | 23 +- src/components/ai-agent/ModelSelector.tsx | 13 +- src/components/ai-agent/ServiceCard.tsx | 20 +- src/components/ai-agent/XrayPanel.tsx | 14 +- src/components/ai-agent/hooks/useChat.ts | 4 + src/components/ai-agent/types.ts | 6 + src/components/ai-agent/utils.ts | 15 + .../lib/validation/aiAgentConfigSchemas.ts | 2 + src/server/services/ai/context/gatherer.ts | 1 + .../services/ai/orchestration/orchestrator.ts | 28 +- .../prompts/__tests__/sectionRegistry.test.ts | 162 ++++++++-- src/server/services/ai/prompts/builder.ts | 1 + .../services/ai/prompts/sectionRegistry.ts | 19 +- .../ai/prompts/sections/foundations.ts | 53 +-- .../ai/prompts/sections/investigation.ts | 302 +++++++++++++++--- .../services/ai/prompts/sections/reference.ts | 19 +- .../services/ai/prompts/sections/safety.ts | 27 +- src/server/services/ai/providers/anthropic.ts | 10 + src/server/services/ai/providers/gemini.ts | 12 + src/server/services/ai/providers/openai.ts | 19 ++ src/server/services/ai/service.ts | 16 + .../tools/github/__tests__/updateFile.test.ts | 168 ++++++++++ .../services/ai/tools/github/updateFile.ts | 47 +++ .../services/ai/tools/k8s/queryDatabase.ts | 2 +- src/server/services/ai/types/provider.ts | 6 + .../services/ai/utils/modelTransformation.ts | 14 +- src/server/services/aiAgent.ts | 16 + src/server/services/aiAgentConfig.ts | 1 - src/server/services/types/aiAgent.ts | 7 + src/shared/types/aiChat.ts | 4 + 31 files changed, 904 insertions(+), 141 deletions(-) create mode 100644 src/server/services/ai/tools/github/__tests__/updateFile.test.ts diff --git a/src/app/api/v2/ai/chat/[buildUuid]/route.ts b/src/app/api/v2/ai/chat/[buildUuid]/route.ts index d26eb484..1320a536 100644 --- a/src/app/api/v2/ai/chat/[buildUuid]/route.ts +++ b/src/app/api/v2/ai/chat/[buildUuid]/route.ts @@ -115,9 +115,15 @@ const postHandler = async (req: NextRequest, { params }: { params: { buildUuid: const { readable, writable } = new TransformStream(); const writer = writable.getWriter(); const encoder = new TextEncoder(); + let writerClosed = false; const sendEvent = (data: AIChatSSEEvent | SSEErrorEvent) => { - writer.write(encoder.encode(`data: ${JSON.stringify(data)}\n\n`)).catch(() => {}); + if (writerClosed) return; + try { + writer.write(encoder.encode(`data: ${JSON.stringify(data)}\n\n`)).catch(() => {}); + } catch { + writerClosed = true; + } }; const MAX_STORED_JSON_LENGTH = 8000; @@ -132,6 +138,7 @@ const postHandler = async (req: NextRequest, { params }: { params: { buildUuid: }; req.signal.addEventListener('abort', () => { + writerClosed = true; try { writer.close(); } catch { @@ -334,6 +341,10 @@ const postHandler = async (req: NextRequest, { params }: { params: { buildUuid: iterations: e.iterations, totalToolCalls: e.totalToolCalls, totalDurationMs: e.totalDurationMs, + inputTokens: e.inputTokens, + outputTokens: e.outputTokens, + inputCostPerMillion: e.inputCostPerMillion, + outputCostPerMillion: e.outputCostPerMillion, }; } } catch { @@ -464,6 +475,7 @@ const postHandler = async (req: NextRequest, { params }: { params: { buildUuid: modelName: 'AI model', }); } finally { + writerClosed = true; try { await writer.close(); } catch { diff --git a/src/components/ai-agent/ChatContainer.tsx b/src/components/ai-agent/ChatContainer.tsx index da2b4352..53c6d793 100644 --- a/src/components/ai-agent/ChatContainer.tsx +++ b/src/components/ai-agent/ChatContainer.tsx @@ -25,7 +25,7 @@ import { ChatInput } from './ChatInput'; import { SuggestedPrompts } from './SuggestedPrompts'; import { MessageList } from './MessageList'; import { globalStyles } from './styles'; -import { getFollowUpSuggestions } from './utils'; +import { getFollowUpSuggestions, computeCost, formatCost } from './utils'; import type { ServiceInvestigationResult } from './types'; interface ChatContainerProps { @@ -82,6 +82,26 @@ export function ChatContainer({ buildUuid }: ChatContainerProps) { [messages, loading, streaming] ); + const sessionTotalCost = useMemo(() => { + let total = 0; + let hasCost = false; + for (const msg of messages) { + if (msg.debugMetrics?.inputTokens != null) { + const cost = computeCost( + msg.debugMetrics.inputTokens, + msg.debugMetrics.outputTokens || 0, + msg.debugMetrics.inputCostPerMillion, + msg.debugMetrics.outputCostPerMillion + ); + if (cost != null) { + total += cost; + hasCost = true; + } + } + } + return hasCost ? formatCost(total) : null; + }, [messages]); + if (!mounted) { return (
@@ -102,6 +122,7 @@ export function ChatContainer({ buildUuid }: ChatContainerProps) { hasMessages={messages.length > 0} onLabelClick={handleLabelClick} xrayMode={xrayMode} + sessionTotalCost={sessionTotalCost} /> diff --git a/src/components/ai-agent/ModelSelector.tsx b/src/components/ai-agent/ModelSelector.tsx index 6723c8f4..3e2537d8 100644 --- a/src/components/ai-agent/ModelSelector.tsx +++ b/src/components/ai-agent/ModelSelector.tsx @@ -27,6 +27,7 @@ interface ModelSelectorProps { hasMessages: boolean; onLabelClick?: () => void; xrayMode?: boolean; + sessionTotalCost?: string | null; } export function ModelSelector({ @@ -38,6 +39,7 @@ export function ModelSelector({ hasMessages, onLabelClick, xrayMode, + sessionTotalCost, }: ModelSelectorProps) { return ( <> @@ -57,7 +59,7 @@ export function ModelSelector({ } labelPlacement="outside-left" size="sm" - className="w-60" + className="w-72" variant="bordered" selectedKeys={selectedModel ? [`${selectedModel.provider}:${selectedModel.modelId}`] : []} onSelectionChange={(keys) => { @@ -66,8 +68,10 @@ export function ModelSelector({ }} isDisabled={loading} classNames={{ - label: 'text-xs text-gray-600 font-semibold', + label: 'text-xs text-gray-600 font-semibold whitespace-nowrap', trigger: 'border-gray-200 hover:border-gray-300', + value: 'text-ellipsis overflow-visible', + innerWrapper: 'overflow-visible', }} > {availableModels.map((model) => ( @@ -77,6 +81,11 @@ export function ModelSelector({ )}
+ {xrayMode && sessionTotalCost && ( + + Session: {sessionTotalCost} + + )} {hasMessages && ( + {service.keyError.length > 200 && ( + + )}
diff --git a/src/components/ai-agent/XrayPanel.tsx b/src/components/ai-agent/XrayPanel.tsx
index 9f9bb6c9..8107f068 100644
--- a/src/components/ai-agent/XrayPanel.tsx
+++ b/src/components/ai-agent/XrayPanel.tsx
@@ -16,7 +16,7 @@
 
 import React from 'react';
 import { Chip } from '@heroui/react';
-import { formatDuration } from './utils';
+import { formatDuration, computeCost, formatCost } from './utils';
 import type { DebugContextData, DebugMetrics } from './types';
 
 interface XrayContextPanelProps {
@@ -63,6 +63,18 @@ export function XrayContextPanel({ debugContext, debugMetrics }: XrayContextPane
             iterations: {debugMetrics.iterations}
             tool calls: {debugMetrics.totalToolCalls}
             duration: {formatDuration(debugMetrics.totalDurationMs)}
+            {debugMetrics.inputTokens != null && (
+              tokens: {(debugMetrics.inputTokens + (debugMetrics.outputTokens || 0)).toLocaleString()} ({debugMetrics.inputTokens.toLocaleString()} in / {(debugMetrics.outputTokens || 0).toLocaleString()} out)
+            )}
+            {debugMetrics.inputTokens != null && (() => {
+              const cost = computeCost(
+                debugMetrics.inputTokens!,
+                debugMetrics.outputTokens || 0,
+                debugMetrics.inputCostPerMillion,
+                debugMetrics.outputCostPerMillion
+              );
+              return cost != null ? cost: {formatCost(cost)} : null;
+            })()}
           
)} diff --git a/src/components/ai-agent/hooks/useChat.ts b/src/components/ai-agent/hooks/useChat.ts index 2f0f9eb8..42c5c7b3 100644 --- a/src/components/ai-agent/hooks/useChat.ts +++ b/src/components/ai-agent/hooks/useChat.ts @@ -251,6 +251,10 @@ export function useChat({ buildUuid, selectedModel }: UseChatOptions) { iterations: data.iterations, totalToolCalls: data.totalToolCalls, totalDurationMs: data.totalDurationMs, + inputTokens: data.inputTokens, + outputTokens: data.outputTokens, + inputCostPerMillion: data.inputCostPerMillion, + outputCostPerMillion: data.outputCostPerMillion, }; setDebugMetrics(localDebugMetrics); } else if (data.type === 'chunk') { diff --git a/src/components/ai-agent/types.ts b/src/components/ai-agent/types.ts index e658ea77..332244dd 100644 --- a/src/components/ai-agent/types.ts +++ b/src/components/ai-agent/types.ts @@ -103,6 +103,8 @@ export interface ModelOption { displayName: string; default: boolean; maxTokens: number; + inputCostPerMillion?: number; + outputCostPerMillion?: number; } export interface DebugToolData { @@ -129,4 +131,8 @@ export interface DebugMetrics { iterations: number; totalToolCalls: number; totalDurationMs: number; + inputTokens?: number; + outputTokens?: number; + inputCostPerMillion?: number; + outputCostPerMillion?: number; } diff --git a/src/components/ai-agent/utils.ts b/src/components/ai-agent/utils.ts index 4d44cc67..f02cdc99 100644 --- a/src/components/ai-agent/utils.ts +++ b/src/components/ai-agent/utils.ts @@ -16,6 +16,21 @@ import type { StructuredDebugResponse, EvidenceItem, ServiceInvestigationResult } from './types'; +export function computeCost( + inputTokens: number, + outputTokens: number, + inputCostPerMillion?: number, + outputCostPerMillion?: number +): number | null { + if (inputCostPerMillion == null || outputCostPerMillion == null) return null; + return (inputTokens * inputCostPerMillion + outputTokens * outputCostPerMillion) / 1_000_000; +} + +export function formatCost(cost: number): string { + if (cost < 0.01) return `$${cost.toFixed(4)}`; + return `$${cost.toFixed(2)}`; +} + export function formatDuration(durationMs?: number): string { if (durationMs === undefined || durationMs === null) return ''; diff --git a/src/server/lib/validation/aiAgentConfigSchemas.ts b/src/server/lib/validation/aiAgentConfigSchemas.ts index 4180ccf3..c39dad69 100644 --- a/src/server/lib/validation/aiAgentConfigSchemas.ts +++ b/src/server/lib/validation/aiAgentConfigSchemas.ts @@ -36,6 +36,8 @@ export const aiAgentConfigSchema = { enabled: { type: 'boolean' }, default: { type: 'boolean' }, maxTokens: { type: 'integer', minimum: 1 }, + inputCostPerMillion: { type: 'number', minimum: 0 }, + outputCostPerMillion: { type: 'number', minimum: 0 }, }, required: ['id', 'displayName', 'enabled', 'default', 'maxTokens'], }, diff --git a/src/server/services/ai/context/gatherer.ts b/src/server/services/ai/context/gatherer.ts index aed619af..afbc5be2 100644 --- a/src/server/services/ai/context/gatherer.ts +++ b/src/server/services/ai/context/gatherer.ts @@ -171,6 +171,7 @@ export default class AIAgentContextService extends BaseService { latestCommit: build.pullRequest.latestCommit, fullName: build.pullRequest.fullName, commentId: build.pullRequest.commentId, + labels: build.pullRequest.labels || [], }, environment: { id: build.environment.id, diff --git a/src/server/services/ai/orchestration/orchestrator.ts b/src/server/services/ai/orchestration/orchestrator.ts index ca0484de..9ad337d3 100644 --- a/src/server/services/ai/orchestration/orchestrator.ts +++ b/src/server/services/ai/orchestration/orchestrator.ts @@ -38,6 +38,8 @@ export interface OrchestrationResult { iterations: number; toolCalls: number; duration: number; + inputTokens: number; + outputTokens: number; }; } @@ -60,6 +62,8 @@ export class ToolOrchestrator { const startTime = Date.now(); let iteration = 0; let totalToolCalls = 0; + let totalInputTokens = 0; + let totalOutputTokens = 0; let fullResponse = ''; const protection = this.loopDetector.getProtection(); @@ -74,7 +78,7 @@ export class ToolOrchestrator { success: false, error: 'Operation cancelled by user', cancelled: true, - metrics: this.buildMetrics(iteration, totalToolCalls, startTime), + metrics: this.buildMetrics(iteration, totalToolCalls, startTime, totalInputTokens, totalOutputTokens), }; } @@ -110,7 +114,7 @@ export class ToolOrchestrator { success: false, error: 'Provider circuit breaker is open', classifiedError: classified, - metrics: this.buildMetrics(iteration, totalToolCalls, startTime), + metrics: this.buildMetrics(iteration, totalToolCalls, startTime, totalInputTokens, totalOutputTokens), }; } @@ -127,7 +131,7 @@ export class ToolOrchestrator { response: fullResponse || 'The response was interrupted. Here is what was generated before the error.', error: `Stream interrupted: ${error.message}`, classifiedError: createClassifiedError(provider.name, error), - metrics: this.buildMetrics(iteration, totalToolCalls, startTime), + metrics: this.buildMetrics(iteration, totalToolCalls, startTime, totalInputTokens, totalOutputTokens), }; } @@ -138,10 +142,16 @@ export class ToolOrchestrator { success: false, error: error.message || 'Provider error', classifiedError: createClassifiedError(provider.name, error), - metrics: this.buildMetrics(iteration, totalToolCalls, startTime), + metrics: this.buildMetrics(iteration, totalToolCalls, startTime, totalInputTokens, totalOutputTokens), }; } + const usageChunk = chunks.find((c) => c.usage); + if (usageChunk?.usage) { + totalInputTokens += usageChunk.usage.inputTokens; + totalOutputTokens += usageChunk.usage.outputTokens; + } + const llmThinkTime = Date.now() - iterationStartTime; const toolCalls = this.extractToolCalls(chunks); @@ -149,7 +159,7 @@ export class ToolOrchestrator { return { success: true, response: fullResponse, - metrics: this.buildMetrics(iteration, totalToolCalls, startTime), + metrics: this.buildMetrics(iteration, totalToolCalls, startTime, totalInputTokens, totalOutputTokens), }; } @@ -165,7 +175,7 @@ export class ToolOrchestrator { error: `Tool call limit exceeded (${protection.maxToolCalls}). ` + `The investigation is too broad. Try asking about specific services.`, - metrics: this.buildMetrics(iteration, totalToolCalls, startTime), + metrics: this.buildMetrics(iteration, totalToolCalls, startTime, totalInputTokens, totalOutputTokens), }; } @@ -341,7 +351,7 @@ export class ToolOrchestrator { error: `Investigation incomplete - hit iteration limit (${protection.maxIterations}). ` + `This may indicate the issue is complex or unclear from available data.`, - metrics: this.buildMetrics(iteration, totalToolCalls, startTime), + metrics: this.buildMetrics(iteration, totalToolCalls, startTime, totalInputTokens, totalOutputTokens), }; } @@ -349,11 +359,13 @@ export class ToolOrchestrator { return chunks.filter((c) => c.type === 'tool_call' && c.toolCalls).flatMap((c) => c.toolCalls || []); } - private buildMetrics(iterations: number, toolCalls: number, startTime: number) { + private buildMetrics(iterations: number, toolCalls: number, startTime: number, inputTokens = 0, outputTokens = 0) { return { iterations, toolCalls, duration: Date.now() - startTime, + inputTokens, + outputTokens, }; } } diff --git a/src/server/services/ai/prompts/__tests__/sectionRegistry.test.ts b/src/server/services/ai/prompts/__tests__/sectionRegistry.test.ts index 223e8308..4aa918e3 100644 --- a/src/server/services/ai/prompts/__tests__/sectionRegistry.test.ts +++ b/src/server/services/ai/prompts/__tests__/sectionRegistry.test.ts @@ -21,19 +21,17 @@ describe('Prompt Section Registry', () => { describe('assembleBasePrompt', () => { it('excludes safety section', () => { const prompt = assembleBasePrompt(); - expect(prompt).not.toContain('# Security & Safety'); + expect(prompt).not.toContain('Violation Examples'); expect(prompt).not.toContain('# Final Reminder'); }); it('includes all non-safety sections', () => { const prompt = assembleBasePrompt(); - expect(prompt).toContain('You are an SRE agent'); - expect(prompt).toContain('# Communication Style'); + expect(prompt).toContain('developers who are blocked'); expect(prompt).toContain('# Investigation Principles'); - expect(prompt).toContain('## Tool Execution Rules'); - expect(prompt).toContain('# Output Format'); + expect(prompt).toContain('# Output Rules'); expect(prompt).toContain('# Lifecycle Architecture'); - expect(prompt).toContain('# Examples'); + expect(prompt).toContain('# Configuration Architecture'); }); it('excludes sections by id when excludeIds provided', () => { @@ -44,28 +42,24 @@ describe('Prompt Section Registry', () => { it('still includes non-excluded sections when excluding reference', () => { const prompt = assembleBasePrompt(['reference']); - expect(prompt).toContain('You are an SRE agent'); - expect(prompt).toContain('# Communication Style'); + expect(prompt).toContain('developers who are blocked'); expect(prompt).toContain('# Investigation Principles'); - expect(prompt).toContain('## Tool Execution Rules'); - expect(prompt).toContain('# Output Format'); }); it('supports excluding multiple sections', () => { const prompt = assembleBasePrompt(['investigation', 'reference']); - expect(prompt).not.toContain('# Examples'); + expect(prompt).not.toContain(''); expect(prompt).not.toContain('# Lifecycle Architecture'); - expect(prompt).toContain('You are an SRE agent'); - expect(prompt).toContain('# Communication Style'); + expect(prompt).toContain('developers who are blocked'); }); it('returns same result with empty excludeIds as no argument', () => { expect(assembleBasePrompt([])).toBe(assembleBasePrompt()); }); - it('assembles sections in correct order', () => { + it('assembles sections in correct order (reference before foundations before investigation)', () => { const prompt = assembleBasePrompt(); - const markers = ['You are an SRE agent', '# Investigation Principles', '# Lifecycle Architecture']; + const markers = ['# Configuration Architecture', 'developers who are blocked', '# Investigation Principles']; const positions = markers.map((m) => prompt.indexOf(m)); for (let i = 1; i < positions.length; i++) { @@ -82,9 +76,50 @@ describe('Prompt Section Registry', () => { expect(assembled).toContain('Verification Protocol'); expect(assembled).toContain('Two-Step Verification'); expect(assembled).toContain('Lifecycle Architecture'); - expect(assembled).toContain('# Examples'); + expect(assembled).toContain(''); expect(assembled).toContain('# Multi-Turn Conversation'); }); + + it('contains XML section tags in assembled prompt', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + }); + + it('contains output schema and field rules separation', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain(''); + expect(assembled).toContain(''); + expect(assembled).toContain('## Field Rules'); + }); + + it('examples contain full JSON with example_output tags', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain(''); + expect(assembled).toContain('"type": "investigation_complete"'); + expect(assembled).toContain('"fixesApplied": false'); + expect(assembled).toContain('"fixesApplied": true'); + expect(assembled).toContain('"oldContent"'); + expect(assembled).toContain('"newContent"'); + }); + + it('schema uses lowercase status values', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain('build_failed'); + expect(assembled).toContain('deploy_failed'); + expect(assembled).not.toContain('"status": "BUILD_FAILED'); + expect(assembled).not.toContain('"status": "DEPLOY_FAILED'); + }); }); describe('PROMPT_SECTIONS', () => { @@ -107,6 +142,11 @@ describe('Prompt Section Registry', () => { expect(sorted[sorted.length - 1].id).toBe('safety'); }); + it('positions reference section first', () => { + const sorted = PROMPT_SECTIONS.slice().sort((a, b) => a.order - b.order); + expect(sorted[0].id).toBe('reference'); + }); + it('has non-empty content for each section', () => { for (const section of PROMPT_SECTIONS) { expect(typeof section.content).toBe('string'); @@ -126,11 +166,14 @@ describe('Prompt Section Registry', () => { it('assembled prompt contains all eval-critical phrases', () => { const assembled = assembleBasePrompt(); const requiredPhrases = [ + 'developers who are blocked', + 'reason about what you expect', + '# Output Rules', + 'Greeting, unclear question, need clarification', 'Investigation Principles', 'Hypothesis-Driven', 'Evidence-Based Stopping', 'Fix Application Workflow', - 'MAX 1 call', 'investigation_complete', 'fixesApplied', 'canAutoFix', @@ -138,7 +181,6 @@ describe('Prompt Section Registry', () => { 'Compare States', 'Multi-Repo Architecture', 'Two-Step Verification', - '# Constraints', 'Multi-Turn Conversation', 'Challenge Responses', 'Confidence Levels', @@ -149,14 +191,98 @@ describe('Prompt Section Registry', () => { } }); + it('assembled prompt does NOT contain V1 CoT suppression', () => { + const assembled = assembleBasePrompt(); + expect(assembled).not.toContain('Execute tools immediately without announcing intent'); + expect(assembled).not.toContain('Analysis AFTER results, not before'); + }); + + it('assembled prompt removes hard tool limit', () => { + const assembled = assembleBasePrompt(); + expect(assembled).not.toContain('Hard limit: 20 tool calls'); + }); + it('safety section contains required safety rules', () => { const safetySection = PROMPT_SECTIONS.find((s) => s.id === 'safety'); expect(safetySection).toBeDefined(); expect(safetySection!.content).toBe(SAFETY_SECTION); expect(safetySection!.content).toContain('User Consent'); expect(safetySection!.content).toContain('Surgical Changes'); + expect(safetySection!.content).toContain('Scope Boundaries'); expect(safetySection!.content).toContain('Path Verification'); expect(safetySection!.content).toContain('Compare States'); + expect(safetySection!.content).toContain('Violation Examples'); + }); + + it('safety section contains Content Integrity rule', () => { + const safetySection = PROMPT_SECTIONS.find((s) => s.id === 'safety'); + expect(safetySection!.content).toContain('Content Integrity'); + expect(safetySection!.content).toContain('EXACT content returned by get_file'); + }); + + it('safety section contains XML tags', () => { + const safetySection = PROMPT_SECTIONS.find((s) => s.id === 'safety'); + expect(safetySection!.content).toContain(''); + expect(safetySection!.content).toContain(''); + }); + + it('investigation section fix workflow contains verification step', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain('does your new_content differ from the original in ONLY the intended lines'); + }); + + it('investigation section contains unrelated-changes negative example', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain('commented-out service configuration'); + }); + + it('safety section contains No Fabrication rule', () => { + const safetySection = PROMPT_SECTIONS.find((s) => s.id === 'safety'); + expect(safetySection!.content).toContain('No Fabrication'); + }); + + it('investigation section contains Insufficient Evidence rule', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain('Insufficient Evidence'); + expect(assembled).toContain('do NOT fabricate a root cause from config analysis alone'); + }); + + it('investigation section contains cite-then-conclude evidence pattern', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain('cite-then-conclude'); + expect(assembled).toContain('first cite the specific error message from your tool results'); + }); + + it('investigation section contains external knowledge restriction', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain('External Knowledge Restriction'); + expect(assembled).toContain('Do not apply general knowledge'); + }); + + it('investigation section explicitly permits saying I dont know', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain("I don't have enough information to determine the root cause"); + }); + + it('investigation section contains fabricated-diagnosis negative example', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain('finds 0 pods running and no error messages'); + }); + + it('canAutoFix rule requires actual error message', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain('a specific error message from logs/K8s/build output points to the problem'); + }); + + it('has compound failure example', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain('cascading effect'); + }); + + it('has everything-healthy example', () => { + const assembled = assembleBasePrompt(); + expect(assembled).toContain('All 3 services'); + expect(assembled).toContain('What specific issue are you seeing'); }); }); }); diff --git a/src/server/services/ai/prompts/builder.ts b/src/server/services/ai/prompts/builder.ts index 62802777..657f49a6 100644 --- a/src/server/services/ai/prompts/builder.ts +++ b/src/server/services/ai/prompts/builder.ts @@ -367,6 +367,7 @@ ${serverSections} Build: ${debugContext.buildUuid} | Status: ${lc.build.status} | Namespace: ${lc.build.namespace} PR: #${lc.pullRequest.number || 'N/A'} "${lc.pullRequest.title || 'N/A'}" by ${lc.pullRequest.username || 'N/A'} +Labels: ${lc.pullRequest.labels?.length ? lc.pullRequest.labels.join(', ') : 'none'} Repo: ${lc.pullRequest.fullName} @ ${lc.pullRequest.branch} (base: ${lc.pullRequest.baseBranch || 'N/A'}) SHA: ${lc.pullRequest.latestCommit || lc.build.sha || 'N/A'} ${ diff --git a/src/server/services/ai/prompts/sectionRegistry.ts b/src/server/services/ai/prompts/sectionRegistry.ts index 95f980a1..c210b35a 100644 --- a/src/server/services/ai/prompts/sectionRegistry.ts +++ b/src/server/services/ai/prompts/sectionRegistry.ts @@ -25,22 +25,23 @@ export interface PromptSection { export const PROMPT_SECTIONS: PromptSection[] = [ { - id: 'foundations', - content: FOUNDATIONS_SECTION, + id: 'reference', + content: REFERENCE_SECTION, order: 1, - rationale: 'Agent identity, communication style, tool rules, and efficiency establish behavioral frame', + rationale: + 'Domain knowledge first — long-context research shows 30% improvement when reference data precedes instructions', }, { - id: 'investigation', - content: INVESTIGATION_SECTION, + id: 'foundations', + content: FOUNDATIONS_SECTION, order: 2, - rationale: 'Investigation strategy, debug/fix workflows, output format, and examples define operational patterns', + rationale: 'Agent identity, communication style, and constraints after domain context is established', }, { - id: 'reference', - content: REFERENCE_SECTION, + id: 'investigation', + content: INVESTIGATION_SECTION, order: 3, - rationale: 'Configuration architecture and domain knowledge provide reference material', + rationale: 'Investigation methodology, output format, and examples define operational patterns', }, { id: 'safety', diff --git a/src/server/services/ai/prompts/sections/foundations.ts b/src/server/services/ai/prompts/sections/foundations.ts index b516ead7..891f5a12 100644 --- a/src/server/services/ai/prompts/sections/foundations.ts +++ b/src/server/services/ai/prompts/sections/foundations.ts @@ -14,11 +14,8 @@ * limitations under the License. */ -// Tier 1 (Static System Prompt): This content is assembled once via assembleBasePrompt() -// and remains constant across all turns in a conversation. For Gemini, this enables -// context caching. All dynamic/per-turn content belongs in Tier 2 (buildEnvironmentContext -// in builder.ts), which is injected per-message. -export const FOUNDATIONS_SECTION = `You are an SRE agent specializing in ephemeral environment troubleshooting. +export const FOUNDATIONS_SECTION = ` +You are an SRE debugging agent for Lifecycle, a platform that creates ephemeral Kubernetes environments from pull requests. Your users are developers who are blocked — their environment isn't working and they need to get back to testing. Your job is to identify why and give them a clear path to resolution. # Primary Objective @@ -26,20 +23,34 @@ Identify root causes by comparing desired config state vs actual runtime state, # Capabilities & Instructions -- **Database = Truth:** DB status is always authoritative. Call tools to verify — initial context may be stale. -- **Data Reuse (HIGHEST PRIORITY):** Query once, reference throughout. The injected environment context already contains lifecycle.yaml summary and service statuses — use that data directly. Only call get_file for a file you have NOT already seen in context or prior tool results. Never re-fetch after read-only operations. -- **Verify When Needed:** If the injected summary lacks detail for a specific service, read that service's config via get_file. Do NOT re-read lifecycle.yaml if a summary is already in context. -- **Root Cause Focus:** Compare DESIRED (config files) vs ACTUAL (runtime) for root cause identification. -- **Parallel Execution:** Execute independent tool calls in parallel. Targeted over exhaustive — investigate specific failing services, not all. +## Data Reuse -## Tool Execution Rules +The injected environment context contains deployment statuses, service health, and K8s state gathered at the timestamp shown. Use this data directly. Only call tools for information not already in context or when the user indicates state has changed. -Execute tools immediately without announcing intent. Call tools directly using the function calling mechanism. Analysis AFTER results, not before. No pseudocode. Avoid code generation unless explicitly requested, but always produce the required JSON output format when instructed. +## Context Freshness + +The injected context reflects current DB state at the \`gathered at\` timestamp. Trust it unless the user indicates something changed ("I just pushed", "I redeployed"). When in doubt about staleness, verify with a single targeted query. + +## Verification + +If the injected summary lacks detail for a specific service, read that service's config via get_file. Reference data from injected context and prior tool results directly rather than re-fetching. + +## Root Cause Focus + +Compare DESIRED (config files) vs ACTUAL (runtime) for root cause identification. + +## Parallel Execution + +Execute independent tool calls in parallel. Targeted over exhaustive — investigate specific failing services, not all. + +## Reasoning + +Before calling tools, briefly reason about what you expect to find. Use evidence from tool results to confirm or refute your hypothesis. # Communication Style -- **Professional & Direct:** Concise tone, <5 lines when practical -- **Minimal Output:** No Chitchat — skip filler and preambles +- Get to the point. Lead with findings, not process descriptions. +- Professional and concise — under 5 lines when practical. - GitHub-flavored Markdown. Clarity over brevity when they conflict. - Tools for actions, text only for user communication. @@ -62,11 +73,11 @@ When results are large or truncated: For uncertain diagnoses (ambiguous logs, partial data, multiple possible causes): state your confidence level, identify what additional evidence would confirm or refute, and gather that evidence before concluding. -# Constraints +# Efficiency -- Hard limit: 20 tool calls per turn. Hit limit → emit partial results. -- Each tool MAX 1 call with same arguments per conversation. Error/not found → move on. -- K8s: ONE call per resource type in the namespace. Use label_selector=lc-service={serviceName} to scope when investigating a specific service. -- Logs: ONE call per pod. If get_pod_logs fails, do not retry. Move on. -- Never re-fetch data you already have — this includes data in the injected environment context. -- Stuck → output what you know + ask user.`; +Be efficient with tool calls. The system enforces a maximum, but most investigations should complete in 5-15 calls. Prefer targeted queries over broad scans. +Each tool should be called at most once with the same arguments per conversation. If a call errors or returns not found, move on. +K8s: ONE call per resource type in the namespace. Use label_selector=lc-service={serviceName} to scope when investigating a specific service. +Logs: ONE call per pod. If get_pod_logs fails, move on. +When stuck, output what you know and ask the user. +`; diff --git a/src/server/services/ai/prompts/sections/investigation.ts b/src/server/services/ai/prompts/sections/investigation.ts index f3b21655..179d18af 100644 --- a/src/server/services/ai/prompts/sections/investigation.ts +++ b/src/server/services/ai/prompts/sections/investigation.ts @@ -14,31 +14,38 @@ * limitations under the License. */ -export const INVESTIGATION_SECTION = `# Investigation Principles +export const INVESTIGATION_SECTION = ` +# Investigation Principles Vague messages ("something seems wrong", "help") — ask for clarification before investigating. **Hypothesis-Driven Flow:** Read injected context (build status, service statuses, K8s state) → form a hypothesis from symptoms → test with targeted tool calls → confirm with evidence or pivot. When a hypothesis is wrong, explain why you're pivoting. -**Evidence-Based Stopping:** Stop investigating once root cause is confirmed with supporting evidence. Do not continue reading files, checking configs, or querying K8s after confirmation. +**Evidence-Based Stopping:** Once you confirm the primary root cause, briefly check if other failing services share the same cause or have independent issues. Report the primary failure in depth and others briefly. **Priority:** Build failures before deploy failures. Investigate the most critical failing service first; briefly mention others. BUILD_FAILED with clear error in logs = sufficient root cause — no need to also check K8s. -**Data Reuse:** Injected context contains environment state — read it before calling ANY tools. Don't re-fetch deploy statuses already provided. Start with recent log lines (tail); request more only if error not found. See Multi-Turn Conversation section for follow-up behavior. +**Data Reuse:** Injected context contains environment state — read it before calling ANY tools. Start with recent log lines (tail); request more only if error not found. See Multi-Turn Conversation section for follow-up behavior. -**Evidence Chains:** Every diagnostic conclusion must trace back to specific log lines, config values, or K8s state observations. Cite the strongest piece of evidence, summarized (not exact quotes). +**Evidence Chains (cite-then-conclude):** Before stating any diagnosis, first cite the specific error message from your tool results that supports it. If you cannot point to a specific error from logs, K8s events, or build output, you do not have evidence and must not make the diagnosis. Reading a config file and deciding it "looks wrong" based on your general knowledge is NOT evidence — only an actual error message from runtime or build output counts. -**Uncertainty:** When root cause can't be determined from available evidence, state uncertainty explicitly rather than guessing. Describe what was checked and what remains unknown. +**External Knowledge Restriction:** Base your diagnosis ONLY on information from tool results (logs, K8s events, build output, database queries) and injected context. Do not apply general knowledge about how configuration files "should" be structured. A config is only wrong when a tool result contains an error pointing to it. + +**Uncertainty — you are allowed to say "I don't know":** When root cause can't be determined from available evidence, you are explicitly permitted to say "I don't have enough information to determine the root cause." This is always preferable to a wrong diagnosis. Describe what you checked and what remains unknown. + +**Insufficient Evidence:** When your investigation finds no error messages — no build logs with errors, no pod crash reasons, no K8s events — do NOT fabricate a root cause from config analysis alone. Instead, report what you checked, what you found (or didn't find), and state that you couldn't determine the root cause. Common causes of missing runtime artifacts (0 pods, no builds) include missing PR labels, webhook delivery issues, and build triggers — mention these as possibilities without diagnosing one as the cause. **You are the investigator.** Read logs, configs, and files yourself. Report findings directly — never suggest the user investigate. Use ACTUAL paths and values, not placeholders. ## Fix Application Workflow "Fix it for me" = EXPLICIT consent for ONLY that specific issue. -1. Call get_file — fetch current content -2. Call update_file or commit_lifecycle_fix with corrected content -3. Verify success response, extract commit_url -4. Output JSON with fixesApplied: true + commitUrl +1. Call get_file — fetch current content. This is your ground truth. +2. Copy the file content EXACTLY. Change ONLY the lines needed for the fix. Do not remove comments, delete unused sections, reformat, or make any other modifications. +3. Before calling update_file, verify: does your new_content differ from the original in ONLY the intended lines? If you changed anything else, redo step 2. +4. Call update_file or commit_lifecycle_fix with the verified content. +5. Verify success response, extract commit_url. +6. Output JSON with fixesApplied: true + commitUrl. Two-step fix pattern: (1) patch K8s resource for immediate relief, (2) persist in config and commit. lifecycle.yaml fixes: offer to apply and commit. Service repo fixes: describe what to change (cannot commit to other repos). @@ -47,61 +54,260 @@ Agent can perform any operation within the service namespace, never outside. ## Verification Protocol Compare States: DESIRED (config) vs ACTUAL (runtime). Match=intentional, mismatch=bug. Always verify config before diagnosing. + -# Output Format + +# Output Rules -**Conversational:** Greetings, unclear question, everything healthy, need direction, hit investigation limits -**JSON:** Any issue found, specific fixes available, config problems even if pods running +Decide your output format using this decision tree: +- Greeting, unclear question, need clarification → plain text (markdown, 1-5 lines) +- Everything healthy, no issues found → plain text confirmation, ask what they're seeing +- Any issue found OR config problem even if pods running → JSON schema below +- Fix applied successfully → JSON with fixesApplied: true and commitUrl JSON — output ONLY valid JSON (no markdown, no conversational text): -\`\`\`json + + { "type": "investigation_complete", - "summary": "If fixesApplied=false, say 'needs to be fixed' NOT 'has been fixed'", + "summary": "string — if fixesApplied=false say 'needs to be fixed' NOT 'has been fixed'", "fixesApplied": false, - "services": [{ - "serviceName": "name", - "status": "BUILD_FAILED | DEPLOY_FAILED | ERROR | READY", - "issue": "ROOT CAUSE with WHY, not just WHAT", - "filePath": "path/to/file.yaml", - "suggestedFix": "Change from '' to '' in ", - "canAutoFix": true, - "lineNumber": 42, "lineNumberEnd": 42, - "files": [{ "path": "...", "lineNumber": 42, "lineNumberEnd": 42, "oldContent": "...", "newContent": "..." }] - }] + "services": [ + { + "serviceName": "string", + "status": "build_failed | deploy_failed | error | ready", + "issue": "string — ROOT CAUSE with WHY, not just WHAT", + "keyError": "string | undefined — exact error message from logs", + "errorSource": "string | undefined — e.g. 'build_logs', 'pod_logs', 'k8s_events'", + "errorSourceDetail": "string | undefined — e.g. pod name, build job id", + "suggestedFix": "string — actionable fix description", + "canAutoFix": "boolean | undefined", + "filePath": "string | undefined — primary file to fix", + "lineNumber": "number | undefined", + "lineNumberEnd": "number | undefined", + "files": [ + { + "path": "string — file path", + "lineNumber": "number | undefined", + "lineNumberEnd": "number | undefined", + "description": "string | undefined — what the change does", + "oldContent": "string | undefined — current content to replace", + "newContent": "string | undefined — corrected content" + } + ], + "commitUrl": "string | undefined — present only when fixesApplied=true" + } + ], + "repository": { + "owner": "string | undefined", + "name": "string | undefined", + "branch": "string | undefined", + "sha": "string | undefined" + } } -\`\`\` + + +## Field Rules + +**status:** Use lowercase values: \`build_failed\`, \`deploy_failed\`, \`error\`, \`ready\`. These must match the TypeScript union type exactly. -**canAutoFix=true** ONLY when: file read + verified, 100% certain, error logs confirm. False when: user decision needed, uncertainty, missing files. +**canAutoFix=true** ONLY when: (1) a specific error message from logs/K8s/build output points to the problem, (2) file read + verified the fix, (3) 100% certain. False when: no error messages found, config-only analysis, user decision needed, uncertainty, missing files. **fixesApplied=true** ONLY when: called update_file, received success, have commit_url. Otherwise false. +**keyError:** Extract the exact error message from logs. Helps the UI display the root cause prominently. + +**errorSource / errorSourceDetail:** Identify where the error was found (e.g., errorSource="pod_logs", errorSourceDetail="web-build-abc-xyz"). + **Line numbers:** Extract from get_file output prefix. Include lineNumber + lineNumberEnd. If can't find, omit both. +**files array:** Include oldContent and newContent when canAutoFix=true. The UI renders these as a diff view. Each entry represents one file change. + +**suggestedFix patterns:** For single-line changes, use the pattern: \`Change from '' to '' in \`. The UI parses this pattern to render inline diffs. For multi-line changes, use the files array with oldContent/newContent instead. + +**commitUrl:** Only include after successfully calling update_file or commit_lifecycle_fix and receiving a commit URL in the response. + +**repository:** Include when available from the injected context. Used by the UI to build GitHub links for file paths. + + + # Examples -user: Why did my web service build fail? +Why did my web service build fail? + Hypothesis: Injected context shows web BUILD_FAILED. Likely a dockerfile or dependency issue. -model: [get_pod_logs: pod="web-build-xyz"] → logs show "COPY failed: file not found ./src/index.ts" -[get_file: lifecycle.yaml] → dockerfilePath references ./src/index.ts but package.json main is ./dist/index.js +Action: [get_pod_logs: pod="web-build-xyz"] → logs show "COPY failed: file not found ./src/index.ts" +Action: [get_file: lifecycle.yaml] → dockerfile references ./src/index.ts but package.json main is ./dist/index.js Root cause confirmed: dockerfile references wrong path. Stop investigating. -Outputs JSON: dockerfilePath mismatch, suggestedFix to update path. + + +{ + "type": "investigation_complete", + "summary": "Web service build failed because the Dockerfile copies ./src/index.ts which does not exist — the correct entrypoint is ./dist/index.js. This needs to be fixed in lifecycle.yaml.", + "fixesApplied": false, + "services": [ + { + "serviceName": "web", + "status": "build_failed", + "issue": "Dockerfile COPY references ./src/index.ts but the actual entrypoint is ./dist/index.js. The build fails at the COPY step because the source file does not exist.", + "keyError": "COPY failed: file not found ./src/index.ts", + "errorSource": "build_logs", + "errorSourceDetail": "web-build-xyz", + "suggestedFix": "Change dockerfilePath entrypoint from './src/index.ts' to './dist/index.js' in lifecycle.yaml", + "canAutoFix": true, + "filePath": "lifecycle.yaml", + "lineNumber": 12, + "lineNumberEnd": 12, + "files": [ + { + "path": "lifecycle.yaml", + "lineNumber": 12, + "lineNumberEnd": 12, + "description": "Fix Dockerfile entrypoint path", + "oldContent": " entrypoint: ./src/index.ts", + "newContent": " entrypoint: ./dist/index.js" + } + ] + } + ] +} + -user: My environment looks broken, what's going on? +My environment looks broken, what's going on? + Hypothesis: Injected context shows api DEPLOY_FAILED, possible resource or image issue. -model: [get_k8s_resources: pods, namespace="env-abc"] → api pod shows OOMKilled (256Mi limit) +Action: [get_k8s_resources: pods, namespace="env-abc"] → api pod shows OOMKilled (256Mi limit) Root cause confirmed: memory limit too low. Stop — don't also check configs or other healthy services. -Outputs JSON: api OOMKilled, increase resources.limits.memory in values file. + + +{ + "type": "investigation_complete", + "summary": "The api service is crashing due to OOMKilled — the memory limit of 256Mi is too low. This needs to be fixed in the Helm values file.", + "fixesApplied": false, + "services": [ + { + "serviceName": "api", + "status": "deploy_failed", + "issue": "Pod is being OOMKilled because the memory limit (256Mi) is insufficient for this service. Kubernetes terminates the container when it exceeds the limit.", + "keyError": "OOMKilled", + "errorSource": "k8s_events", + "errorSourceDetail": "api-deployment-abc-xyz", + "suggestedFix": "Change resources.limits.memory from '256Mi' to '512Mi' in helm/values.yaml", + "canAutoFix": true, + "filePath": "helm/values.yaml", + "lineNumber": 28, + "lineNumberEnd": 28, + "files": [ + { + "path": "helm/values.yaml", + "lineNumber": 28, + "lineNumberEnd": 28, + "description": "Increase memory limit to prevent OOMKill", + "oldContent": " memory: 256Mi", + "newContent": " memory: 512Mi" + } + ] + } + ] +} + + + + +Fix the dockerfile path in lifecycle.yaml + +Action: [get_file: lifecycle.yaml] → read current content (147 lines), identified wrong path on line 42 +Action: Copied original content, changed ONLY line 42. Verified: 146 of 147 lines unchanged. No comments removed, no reformatting. +Action: [commit_lifecycle_fix: path="lifecycle.yaml", content="..."] → success, commit URL returned +Fix applied successfully. + + +{ + "type": "investigation_complete", + "summary": "Fixed the dockerfile path in lifecycle.yaml. The change has been committed.", + "fixesApplied": true, + "services": [ + { + "serviceName": "web", + "status": "build_failed", + "issue": "Dockerfile path referenced a non-existent file ./src/index.ts instead of ./dist/index.js.", + "suggestedFix": "Change dockerfile entrypoint from './src/index.ts' to './dist/index.js' in lifecycle.yaml", + "canAutoFix": true, + "filePath": "lifecycle.yaml", + "lineNumber": 12, + "lineNumberEnd": 12, + "files": [ + { + "path": "lifecycle.yaml", + "lineNumber": 12, + "lineNumberEnd": 12, + "description": "Fix Dockerfile entrypoint path", + "oldContent": " entrypoint: ./src/index.ts", + "newContent": " entrypoint: ./dist/index.js" + } + ], + "commitUrl": "https://github.com/org/repo/commit/abc123" + } + ] +} + + + + +My environment is completely broken, nothing works + +Hypothesis: Injected context shows api DEPLOY_FAILED and worker BUILD_FAILED. Two services failing — could be related or independent. +Action: [get_pod_logs: pod="worker-build-xyz"] → logs show "npm ERR! Missing dependency: @org/shared-lib@2.0" +Root cause for worker: missing dependency in package.json. +Action: [get_k8s_resources: pods, namespace="env-abc"] → api pod shows CrashLoopBackOff, logs show "Error: connect ECONNREFUSED worker:3000" +api failure is downstream — it depends on worker, which failed to build. +Primary root cause: worker build failure (missing dependency). api failure is a cascading effect. + + +{ + "type": "investigation_complete", + "summary": "Worker build failed due to a missing dependency (@org/shared-lib@2.0), which caused a cascading failure in the api service. Fix the worker dependency first — api will recover once worker is running.", + "fixesApplied": false, + "services": [ + { + "serviceName": "worker", + "status": "build_failed", + "issue": "Build fails because @org/shared-lib@2.0 is not in package.json dependencies. This is the primary root cause — the api service depends on worker and fails downstream.", + "keyError": "npm ERR! Missing dependency: @org/shared-lib@2.0", + "errorSource": "build_logs", + "errorSourceDetail": "worker-build-xyz", + "suggestedFix": "Add @org/shared-lib@2.0 to dependencies in package.json", + "canAutoFix": false, + "filePath": "package.json" + }, + { + "serviceName": "api", + "status": "deploy_failed", + "issue": "CrashLoopBackOff because it cannot connect to worker:3000. This is a cascading effect of the worker build failure — not an independent issue.", + "keyError": "Error: connect ECONNREFUSED worker:3000", + "errorSource": "pod_logs", + "errorSourceDetail": "api-deployment-abc-xyz", + "suggestedFix": "No action needed — api will recover once the worker build is fixed and deployed", + "canAutoFix": false + } + ] +} + -user: Fix the dockerfile path in lifecycle.yaml -model: [get_file: lifecycle.yaml] -[commit_lifecycle_fix: path="lifecycle.yaml", content="...corrected..."] -Outputs JSON: fixesApplied=true, commitUrl="https://github.com/org/repo/commit/abc123" +Something seems wrong with my environment + +Hypothesis: Check injected context for failures. +Injected context shows: all 3 services READY, all pods Running with ready=desired. +Verify with one fresh K8s check: [get_k8s_resources: pods, namespace="env-abc"] → confirms all healthy. + + +All 3 services (api, web, worker) are running and healthy. What specific issue are you seeing? + @@ -109,17 +315,18 @@ ANTI-PATTERN: Agent finds OOMKilled in pod logs, confirms root cause, then ALSO CORRECT: Stop after OOMKilled confirmation. Report the fix (increase memory limit) immediately. -# Constraints + +ANTI-PATTERN: Agent fixes dockerfilePath on line 42 of lifecycle.yaml, but the committed diff also deletes 28 lines of commented-out service configuration at the bottom of the file. The user only approved the single-line fix. +CORRECT: Copy the entire original file content from get_file verbatim. Change ONLY line 42. Every other line — including comments, blank lines, and disabled sections — must remain byte-for-byte identical. + -- Do not read files beyond what error references point to -- Do not re-fetch K8s state already in injected context -- Do not continue investigating past root cause confirmation -- Do not explore files looking for hypothetical issues when status is healthy -- User challenges healthy status: ONE fresh K8s check (pods + events). If still healthy, stand firm -- Each tool MAX 1 call with same arguments -- All services healthy: verify via injected context, ask what specific issue they see -- When multiple services fail, investigate the most critical first, then briefly address others + +ANTI-PATTERN: Agent finds 0 pods running and no error messages in any logs. Reads lifecycle.yaml and decides the YAML structure "looks wrong" — proposes config changes with canAutoFix=true. No actual error message pointed to the config. The real cause was outside the agent's investigation (e.g., a missing PR label). +CORRECT: Report that no pods were found and no build/deploy errors were found in logs. State that the root cause could not be determined from available evidence. Suggest possible external causes (PR labels, webhook delivery, build triggers) without diagnosing a specific one. + + + # Multi-Turn Conversation **How conversation context works:** The LLM provider (Gemini/Anthropic/OpenAI) manages the full message thread natively. Your tool calls and their results from the current turn are always visible to you. Tool results from recent prior turns are preserved in the conversation history by the observation masker; older tool results may be masked. Additionally, fresh environment context (deployment status, service health) is injected at the start of each new user message via Tier 2. @@ -141,4 +348,5 @@ CORRECT: Stop after OOMKilled confirmation. Report the fix (increase memory limi **Long Conversations:** In conversations exceeding many turns, prioritize the most recent context. If asked about something discussed many turns ago, acknowledge the data may be stale and offer to re-check. -**Proactive Observations:** If during investigation you notice related issues affecting other services, briefly mention them after addressing the user's primary question.`; +**Proactive Observations:** If during investigation you notice related issues affecting other services, briefly mention them after addressing the user's primary question. +`; diff --git a/src/server/services/ai/prompts/sections/reference.ts b/src/server/services/ai/prompts/sections/reference.ts index 94c80cf8..81dbf0ca 100644 --- a/src/server/services/ai/prompts/sections/reference.ts +++ b/src/server/services/ai/prompts/sections/reference.ts @@ -14,7 +14,8 @@ * limitations under the License. */ -export const REFERENCE_SECTION = `# Configuration Architecture +export const REFERENCE_SECTION = ` +# Configuration Architecture **Hierarchy:** lifecycle.yaml = SITEMAP referencing other files. Configuration is DISTRIBUTED — follow references for actual values. - dockerfile → Dockerfiles | helm.valueFiles → Helm values (replicaCount, resources, ports) | helm.chart → Local charts @@ -36,25 +37,16 @@ Lifecycle creates ephemeral environments from Pull Requests. **Source of Truth (ranked):** 1. DB status 2. Config files (Helm values, Dockerfiles) 3. lifecycle.yaml 4. PR comment 5. K8s state 6. Events 7. Logs +**PR Labels:** Lifecycle requires a deploy label (default: \`lifecycle-deploy!\`) on the PR before it will build and deploy an environment. Without this label, no builds or deploys are created — the environment will have 0 pods and no activity. The \`lifecycle-disabled!\` label explicitly prevents deployment. PR labels are shown in the injected context. If you see 0 pods and no builds/deploys, check whether the deploy label is present. + **Build vs Deploy:** Builds first, deploys only if ALL builds succeed. Check builds before deploy issues. **Build system:** buildPipelineId → Codefresh (get_codefresh_logs) | builderEngine → Native K8s (get_k8s_resources + get_pod_logs) | GITHUB type → no build **K8s:** Deployments AND StatefulSets (check both). Labels: lc-service={serviceName}. Namespace: env-{buildUuid}. -## Database Queries - -Batch with relations — ONE call: -\`query_database(table="builds", filters={"uuid": "xyz"}, relations=["pullRequest", "environment", "deploys.[deployable, repository]"])\` -Provides: build info, PR, environment, deploys, deployables, repos. READ-ONLY. - -**Lifecycle logs:** \`get_lifecycle_logs(build_uuid="{buildUuid}")\` — for debugging Lifecycle itself (jobs not starting, environments not creating). - -**Healthy:** All deploys READY/RUNNING in DB + all K8s ready>=1 AND ready==desired + no CrashLoopBackOff/ImagePullBackOff/Error. - # Multi-Repo Architecture -The PR repo branch contains ONLY lifecycle.yaml (environment config) — NOT service source code. Each service's code lives in its OWN repository, shown as "Repo: Owner/name @ branch" in injected context. **Single-repo environments:** The PR repo IS the service repo — use the PR repo directly. @@ -63,4 +55,5 @@ Each service's code lives in its OWN repository, shown as "Repo: Owner/name @ br Use \`get_file\` with the service's repo and branch params to read service files. If a service repo is inaccessible, tell the user explicitly — do not guess at file contents. Always identify which repo a file is from when referencing it. -Never commit fixes to repos other than the PR repo.`; +Never commit fixes to repos other than the PR repo. +`; diff --git a/src/server/services/ai/prompts/sections/safety.ts b/src/server/services/ai/prompts/sections/safety.ts index b68ac20c..3160420c 100644 --- a/src/server/services/ai/prompts/sections/safety.ts +++ b/src/server/services/ai/prompts/sections/safety.ts @@ -14,14 +14,29 @@ * limitations under the License. */ -export const SAFETY_SECTION = `# Security & Safety +export const SAFETY_SECTION = ` +# Security & Safety -- **User Consent:** NEVER auto-apply fixes. JSON first → user clicks "Fix it for me" → apply + show commit URL. -- **Surgical Changes:** ONLY change what was suggested. No formatting, no cleanup, no unrelated fixes. -- **Path Verification:** NEVER suggest file path without verifying it exists via list_directory or get_file. -- **No Assumptions:** Multiple valid options → ASK user. Verify, don't guess. +- **User Consent:** Present findings as JSON first. The user clicks "Fix it for me" before any changes are applied. After applying, show the commit URL. +- **Surgical Changes:** Change ONLY what was suggested. No formatting, no cleanup, no unrelated fixes. +- **Content Integrity:** When modifying a file via update_file, use the EXACT content returned by get_file as your starting point. Apply only the specific fix — do not remove comments, reformat whitespace, delete unused sections, or make any other modifications. Your new_content must be identical to the original except for the targeted fix lines. +- **Scope Boundaries:** You can modify files in the PR repo only. For issues in other service repos, describe the fix but do not attempt to commit. +- **Path Verification:** Before including a filePath in your JSON response, you must have either (a) read it via get_file, (b) seen it in the injected context, or (c) confirmed it exists via list_directory. +- **No Fabrication:** Never diagnose a root cause you cannot support with a specific error message cited from your tool results. If no tool output contains an error pointing to the problem, say "I don't have enough information to determine the root cause." General knowledge about config structure is not evidence. +- **Ambiguity:** When multiple valid options exist, ask the user to choose. + +## Violation Examples + +WRONG: Agent finds a typo in lifecycle.yaml and also reformats indentation while fixing it. +RIGHT: Agent fixes only the typo. No other changes. + +WRONG: Agent sees canAutoFix=true for a resource limit issue and immediately calls update_file. +RIGHT: Agent outputs JSON with the suggestion. User clicks "Fix it for me". Then agent applies. + +WRONG: Agent fixes a typo on line 42 of lifecycle.yaml but the committed diff also deletes 28 lines of commented-out service configuration at the bottom of the file. +RIGHT: Agent fixes only line 42. All other content — including comments, blank lines, and unused sections — remains byte-for-byte identical to the get_file output. # Final Reminder Compare States: DESIRED (config) vs ACTUAL (runtime). Intentional or bug? When in doubt, ASK. -`; +`; diff --git a/src/server/services/ai/providers/anthropic.ts b/src/server/services/ai/providers/anthropic.ts index bbd35de7..235db7d6 100644 --- a/src/server/services/ai/providers/anthropic.ts +++ b/src/server/services/ai/providers/anthropic.ts @@ -79,6 +79,16 @@ export class AnthropicProvider extends BaseLLMProvider { toolCalls, }; } + + if (response.usage) { + yield { + type: 'text', + usage: { + inputTokens: response.usage.input_tokens, + outputTokens: response.usage.output_tokens, + }, + }; + } } formatHistory(messages: ConversationMessage[]): unknown[] { diff --git a/src/server/services/ai/providers/gemini.ts b/src/server/services/ai/providers/gemini.ts index 76055f5f..27055473 100644 --- a/src/server/services/ai/providers/gemini.ts +++ b/src/server/services/ai/providers/gemini.ts @@ -109,6 +109,7 @@ export class GeminiProvider extends BaseLLMProvider { let accumulatedText = ''; const functionCalls: Array = []; let lastCandidate: Candidate | null = null; + let lastRawChunk: any = null; for await (const chunk of stream) { if (signal?.aborted) { @@ -121,6 +122,7 @@ export class GeminiProvider extends BaseLLMProvider { } lastCandidate = candidate; + lastRawChunk = chunk; if (candidate.finishReason === 'STOP' && (!candidate.content?.parts || candidate.content.parts.length === 0)) { getLogger().error( @@ -184,6 +186,16 @@ export class GeminiProvider extends BaseLLMProvider { toolCalls, }; } + + if (lastRawChunk?.usageMetadata) { + yield { + type: 'text', + usage: { + inputTokens: lastRawChunk.usageMetadata.promptTokenCount || 0, + outputTokens: lastRawChunk.usageMetadata.candidatesTokenCount || 0, + }, + }; + } } formatHistory(messages: ConversationMessage[]): unknown[] { diff --git a/src/server/services/ai/providers/openai.ts b/src/server/services/ai/providers/openai.ts index 8dd4ca51..c64066c9 100644 --- a/src/server/services/ai/providers/openai.ts +++ b/src/server/services/ai/providers/openai.ts @@ -63,17 +63,26 @@ export class OpenAIProvider extends BaseLLMProvider { model: this.modelId, max_tokens: options.maxTokens || 250000, stream: true, + stream_options: { include_usage: true }, messages: openaiMessages, tools: tools || undefined, }); let accumulatedToolCalls: AccumulatedToolCall[] = []; + let finalUsage: { prompt_tokens: number; completion_tokens: number } | null = null; for await (const chunk of stream) { if (signal?.aborted) { throw new Error('Request aborted'); } + if (chunk.usage) { + finalUsage = { + prompt_tokens: chunk.usage.prompt_tokens, + completion_tokens: chunk.usage.completion_tokens, + }; + } + const delta = chunk.choices[0]?.delta; if (delta?.content) { @@ -113,6 +122,16 @@ export class OpenAIProvider extends BaseLLMProvider { toolCalls, }; } + + if (finalUsage) { + yield { + type: 'text', + usage: { + inputTokens: finalUsage.prompt_tokens, + outputTokens: finalUsage.completion_tokens, + }, + }; + } } formatHistory(messages: ConversationMessage[]): unknown[] { diff --git a/src/server/services/ai/service.ts b/src/server/services/ai/service.ts index cd3a73bd..f8ae68e2 100644 --- a/src/server/services/ai/service.ts +++ b/src/server/services/ai/service.ts @@ -58,6 +58,10 @@ export interface AIAgentConfig { systemPromptOverride?: string; excludedTools?: string[]; excludedFilePatterns?: string[]; + modelPricing?: { + inputCostPerMillion: number; + outputCostPerMillion: number; + }; } export interface ProcessQueryResult { @@ -67,6 +71,8 @@ export interface ProcessQueryResult { iterations: number; toolCalls: number; duration: number; + inputTokens: number; + outputTokens: number; }; } @@ -81,6 +87,7 @@ export class AIAgentCore { private systemPromptOverride?: string; private excludedTools?: string[]; private excludedFilePatterns?: string[]; + private modelPricing?: { inputCostPerMillion: number; outputCostPerMillion: number }; private mcpToolsLoaded = false; private mcpToolInfos: McpToolInfo[] = []; @@ -103,6 +110,7 @@ export class AIAgentCore { this.systemPromptOverride = config.systemPromptOverride; this.excludedTools = config.excludedTools; this.excludedFilePatterns = config.excludedFilePatterns; + this.modelPricing = config.modelPricing; this.toolRegistry = new ToolRegistry(); this.registerAllTools(); @@ -249,6 +257,14 @@ export class AIAgentCore { iterations: result.metrics.iterations, totalToolCalls: result.metrics.toolCalls, totalDurationMs: duration, + inputTokens: result.metrics.inputTokens, + outputTokens: result.metrics.outputTokens, + ...(this.modelPricing + ? { + inputCostPerMillion: this.modelPricing.inputCostPerMillion, + outputCostPerMillion: this.modelPricing.outputCostPerMillion, + } + : {}), }); getLogger().info( diff --git a/src/server/services/ai/tools/github/__tests__/updateFile.test.ts b/src/server/services/ai/tools/github/__tests__/updateFile.test.ts new file mode 100644 index 00000000..590ad4a8 --- /dev/null +++ b/src/server/services/ai/tools/github/__tests__/updateFile.test.ts @@ -0,0 +1,168 @@ +/** + * Copyright 2025 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { validateDiff, UpdateFileTool } from '../updateFile'; + +describe('validateDiff', () => { + it('allows identical content', () => { + const content = 'line1\nline2\nline3'; + const result = validateDiff(content, content); + expect(result.valid).toBe(true); + expect(result.linesChanged).toBe(0); + expect(result.linesRemoved).toBe(0); + }); + + it('allows small changes (1-3 lines modified)', () => { + const old = 'line1\nline2\nline3\nline4\nline5'; + const updated = 'line1\nchanged\nline3\nline4\nline5'; + const result = validateDiff(old, updated); + expect(result.valid).toBe(true); + expect(result.linesChanged).toBe(1); + }); + + it('allows changes up to 10 lines', () => { + const lines = Array.from({ length: 20 }, (_, i) => `line${i}`); + const oldContent = lines.join('\n'); + const newLines = [...lines]; + for (let i = 0; i < 10; i++) { + newLines[i] = `changed${i}`; + } + const result = validateDiff(oldContent, newLines.join('\n')); + expect(result.valid).toBe(true); + expect(result.linesChanged).toBe(10); + }); + + it('rejects excessive changes (>10 lines changed)', () => { + const lines = Array.from({ length: 20 }, (_, i) => `line${i}`); + const oldContent = lines.join('\n'); + const newLines = [...lines]; + for (let i = 0; i < 11; i++) { + newLines[i] = `changed${i}`; + } + const result = validateDiff(oldContent, newLines.join('\n')); + expect(result.valid).toBe(false); + expect(result.linesChanged).toBe(11); + expect(result.error).toContain('SAFETY ERROR'); + expect(result.error).toContain('changes 11 lines'); + }); + + it('allows small deletions (up to 3 lines removed)', () => { + const old = 'line1\nline2\nline3\nline4\nline5\nline6\nline7'; + const updated = 'line1\nline2\nline3\nline7'; + const result = validateDiff(old, updated); + expect(result.valid).toBe(true); + }); + + it('rejects large deletions (>3 lines removed)', () => { + const lines = Array.from({ length: 50 }, (_, i) => `line${i}`); + const oldContent = lines.join('\n'); + const newContent = lines.slice(0, 20).join('\n'); + const result = validateDiff(oldContent, newContent); + expect(result.valid).toBe(false); + expect(result.linesRemoved).toBe(30); + expect(result.error).toContain('SAFETY ERROR'); + expect(result.error).toContain('removes 30 lines'); + }); + + it('allows adding lines when total changes stay within limit', () => { + const old = 'line1\nline2\nline3'; + const updated = 'line1\nline2\nnew_line\nline3'; + const result = validateDiff(old, updated); + expect(result.valid).toBe(true); + }); +}); + +describe('UpdateFileTool', () => { + const mockOctokit = { request: jest.fn() }; + const mockGithubClient = { + getOctokit: jest.fn().mockResolvedValue(mockOctokit), + isFilePathAllowed: jest.fn().mockReturnValue(true), + validateBranch: jest.fn().mockReturnValue({ valid: true }), + } as any; + + let tool: UpdateFileTool; + + const baseArgs = { + repository_owner: 'org', + repository_name: 'repo', + branch: 'feature-branch', + file_path: 'lifecycle.yaml', + new_content: 'line1\nline2\nchanged\nline4\nline5', + commit_message: 'fix typo', + }; + + beforeEach(() => { + jest.clearAllMocks(); + mockGithubClient.getOctokit.mockResolvedValue(mockOctokit); + mockGithubClient.isFilePathAllowed.mockReturnValue(true); + mockGithubClient.validateBranch.mockReturnValue({ valid: true }); + tool = new UpdateFileTool(mockGithubClient); + }); + + it('skips validation for new files', async () => { + mockOctokit.request.mockRejectedValueOnce(new Error('Not Found')); + mockOctokit.request.mockResolvedValueOnce({ + data: { commit: { sha: 'abc', html_url: 'https://github.com/org/repo/commit/abc' } }, + }); + + const result = await tool.execute(baseArgs); + expect(result.success).toBe(true); + }); + + it('rejects updates that remove too many lines', async () => { + const originalLines = Array.from({ length: 50 }, (_, i) => `line${i}`); + const originalContent = originalLines.join('\n'); + + mockOctokit.request.mockResolvedValueOnce({ + data: { + sha: 'existing-sha', + content: Buffer.from(originalContent).toString('base64'), + }, + }); + + const result = await tool.execute({ + ...baseArgs, + new_content: originalLines.slice(0, 20).join('\n'), + }); + + expect(result.success).toBe(false); + expect(result.error?.code).toBe('DIFF_VALIDATION_FAILED'); + expect(result.agentContent).toContain('SAFETY ERROR'); + }); + + it('allows updates with small changes', async () => { + const originalContent = 'line1\nline2\nline3\nline4\nline5'; + const newContent = 'line1\nline2\nchanged\nline4\nline5'; + + mockOctokit.request + .mockResolvedValueOnce({ + data: { + sha: 'existing-sha', + content: Buffer.from(originalContent).toString('base64'), + }, + }) + .mockResolvedValueOnce({ + data: { commit: { sha: 'new-sha', html_url: 'https://github.com/org/repo/commit/new-sha' } }, + }); + + const result = await tool.execute({ + ...baseArgs, + new_content: newContent, + }); + + expect(result.success).toBe(true); + }); +}); diff --git a/src/server/services/ai/tools/github/updateFile.ts b/src/server/services/ai/tools/github/updateFile.ts index 0b89ca41..f2b24b17 100644 --- a/src/server/services/ai/tools/github/updateFile.ts +++ b/src/server/services/ai/tools/github/updateFile.ts @@ -18,6 +18,42 @@ import { BaseTool } from '../baseTool'; import { ToolResult, ToolSafetyLevel, ConfirmationDetails } from '../../types/tool'; import { GitHubClient } from '../shared/githubClient'; +export function validateDiff( + oldContent: string, + newContent: string +): { valid: boolean; error?: string; linesRemoved: number; linesChanged: number } { + const oldLines = oldContent.split('\n'); + const newLines = newContent.split('\n'); + const linesRemoved = Math.max(0, oldLines.length - newLines.length); + + let linesChanged = 0; + const minLen = Math.min(oldLines.length, newLines.length); + for (let i = 0; i < minLen; i++) { + if (oldLines[i] !== newLines[i]) linesChanged++; + } + linesChanged += Math.abs(oldLines.length - newLines.length); + + if (linesRemoved > 3) { + return { + valid: false, + linesRemoved, + linesChanged, + error: `SAFETY ERROR: Your update removes ${linesRemoved} lines from the original file. Only modify the specific lines needed for the fix. Use the exact content from get_file as your starting point, change only the targeted lines, and resubmit.`, + }; + } + + if (linesChanged > 10) { + return { + valid: false, + linesRemoved, + linesChanged, + error: `SAFETY ERROR: Your update changes ${linesChanged} lines. This exceeds the expected scope for a targeted fix. Use the exact content from get_file as your starting point, change only the targeted lines, and resubmit.`, + }; + } + + return { valid: true, linesRemoved, linesChanged }; +} + export class UpdateFileTool extends BaseTool { static readonly Name = 'update_file'; @@ -92,6 +128,7 @@ export class UpdateFileTool extends BaseTool { const octokit = await this.githubClient.getOctokit('ai-agent-update-file'); let currentFileSha: string | undefined; + let currentFileContent: string | undefined; try { const currentFile = await octokit.request(`GET /repos/${owner}/${repo}/contents/${filePath}`, { ref: branch, @@ -99,12 +136,22 @@ export class UpdateFileTool extends BaseTool { if (currentFile.data && 'sha' in currentFile.data) { currentFileSha = currentFile.data.sha; } + if (currentFile.data && 'content' in currentFile.data) { + currentFileContent = Buffer.from(currentFile.data.content as string, 'base64').toString('utf-8'); + } } catch (error) { currentFileSha = undefined; } const contentToCommit = newContent.replace(/\\n/g, '\n').replace(/\\r/g, '\r').replace(/\\t/g, '\t'); + if (currentFileContent) { + const diffResult = validateDiff(currentFileContent, contentToCommit); + if (!diffResult.valid) { + return this.createErrorResult(diffResult.error!, 'DIFF_VALIDATION_FAILED', false); + } + } + const response = await octokit.request(`PUT /repos/${owner}/${repo}/contents/${filePath}`, { message: `[Lifecycle AI] ${commitMessage}`, content: Buffer.from(contentToCommit).toString('base64'), diff --git a/src/server/services/ai/tools/k8s/queryDatabase.ts b/src/server/services/ai/tools/k8s/queryDatabase.ts index 9ce70b7c..5a088d88 100644 --- a/src/server/services/ai/tools/k8s/queryDatabase.ts +++ b/src/server/services/ai/tools/k8s/queryDatabase.ts @@ -36,7 +36,7 @@ export class QueryDatabaseTool extends BaseTool { filters: { type: 'object', description: - 'WHERE conditions as key-value pairs. Deploy uuid format is "{serviceName}-{buildUuid}" (e.g., "vpii-events-broad-lab-080573"). Use SQL LIKE patterns with % for partial matching: {"uuid": "vpii-events-%"} finds all deploys for service vpii-events. Use "uuid" for builds/deploys, "id" for others. Only use actual table columns as keys.', + 'WHERE conditions as key-value pairs. Deploy uuid format is "{serviceName}-{buildUuid}" (e.g., "vpii-events-broad-lab-080573"). Use SQL LIKE patterns with % for partial matching: {"uuid": "vpii-events-%"} finds all deploys for service vpii-events. Use "uuid" for builds/deploys, "id" for others. Only use actual table columns as keys. IMPORTANT: To find all deploys for a build, query the builds table with relations: ["deploys"], e.g., {"table": "builds", "filters": {"uuid": "my-build-uuid"}, "relations": ["deploys"]}. Do NOT filter deploys by buildId — it is a numeric FK, not the build UUID string.', }, relations: { type: 'array', diff --git a/src/server/services/ai/types/provider.ts b/src/server/services/ai/types/provider.ts index 185b8e84..154fee53 100644 --- a/src/server/services/ai/types/provider.ts +++ b/src/server/services/ai/types/provider.ts @@ -26,6 +26,11 @@ export interface CompletionOptions { temperature?: number; } +export interface TokenUsage { + inputTokens: number; + outputTokens: number; +} + export type StreamChunkType = 'text' | 'tool_call' | 'thinking' | 'error'; export interface StreamChunk { @@ -33,6 +38,7 @@ export interface StreamChunk { content?: string; toolCalls?: ToolCall[]; metadata?: Record; + usage?: TokenUsage; } export interface ModelInfo { diff --git a/src/server/services/ai/utils/modelTransformation.ts b/src/server/services/ai/utils/modelTransformation.ts index 621945ac..4c4c92ca 100644 --- a/src/server/services/ai/utils/modelTransformation.ts +++ b/src/server/services/ai/utils/modelTransformation.ts @@ -14,9 +14,15 @@ * limitations under the License. */ -export function transformProviderModels( - providers: any[] -): Array<{ provider: string; modelId: string; displayName: string; default: boolean; maxTokens: number }> { +export function transformProviderModels(providers: any[]): Array<{ + provider: string; + modelId: string; + displayName: string; + default: boolean; + maxTokens: number; + inputCostPerMillion?: number; + outputCostPerMillion?: number; +}> { return providers .filter((provider: any) => provider.enabled) .flatMap((provider: any) => { @@ -32,6 +38,8 @@ export function transformProviderModels( displayName: model.displayName, default: model.default || false, maxTokens: model.maxTokens, + ...(model.inputCostPerMillion != null ? { inputCostPerMillion: model.inputCostPerMillion } : {}), + ...(model.outputCostPerMillion != null ? { outputCostPerMillion: model.outputCostPerMillion } : {}), })); }); } diff --git a/src/server/services/aiAgent.ts b/src/server/services/aiAgent.ts index 813a3aa3..83890771 100644 --- a/src/server/services/aiAgent.ts +++ b/src/server/services/aiAgent.ts @@ -39,6 +39,7 @@ export default class AIAgentService extends BaseService { private service: AIAgentCore | null = null; private provider: ProviderType = 'anthropic'; private modelId?: string; + private modelPricing?: { inputCostPerMillion: number; outputCostPerMillion: number }; private currentMode: 'investigate' | 'fix' = 'investigate'; private repoFullName?: string; @@ -78,6 +79,13 @@ export default class AIAgentService extends BaseService { this.provider = provider; this.modelId = modelId; + this.modelPricing = + modelConfig.inputCostPerMillion != null && modelConfig.outputCostPerMillion != null + ? { + inputCostPerMillion: modelConfig.inputCostPerMillion, + outputCostPerMillion: modelConfig.outputCostPerMillion, + } + : undefined; getLogger().info(`AI: provider set to ${this.provider} modelId=${this.modelId}`); } else { getLogger().info(`AI: no provider/modelId provided, using defaults`); @@ -91,6 +99,13 @@ export default class AIAgentService extends BaseService { } this.provider = enabledProvider.name as ProviderType; this.modelId = defaultModel.id; + this.modelPricing = + defaultModel.inputCostPerMillion != null && defaultModel.outputCostPerMillion != null + ? { + inputCostPerMillion: defaultModel.inputCostPerMillion, + outputCostPerMillion: defaultModel.outputCostPerMillion, + } + : undefined; getLogger().info(`AI: using default provider=${this.provider} modelId=${this.modelId}`); } @@ -108,6 +123,7 @@ export default class AIAgentService extends BaseService { systemPromptOverride: aiAgentConfig.systemPromptOverride, excludedTools: aiAgentConfig.excludedTools, excludedFilePatterns: aiAgentConfig.excludedFilePatterns, + modelPricing: this.modelPricing, }); } diff --git a/src/server/services/aiAgentConfig.ts b/src/server/services/aiAgentConfig.ts index 411fc0ff..e9e50ab8 100644 --- a/src/server/services/aiAgentConfig.ts +++ b/src/server/services/aiAgentConfig.ts @@ -109,7 +109,6 @@ export default class AIAgentConfigService extends BaseService { if (repoOverride.systemPromptOverride !== undefined) { result.systemPromptOverride = repoOverride.systemPromptOverride; } - const arrayFields: (keyof AIAgentRepoOverride)[] = ['additiveRules', 'excludedTools', 'excludedFilePatterns']; for (const field of arrayFields) { diff --git a/src/server/services/types/aiAgent.ts b/src/server/services/types/aiAgent.ts index 38c15085..560ea0a6 100644 --- a/src/server/services/types/aiAgent.ts +++ b/src/server/services/types/aiAgent.ts @@ -60,6 +60,10 @@ export interface DebugMessage { iterations: number; totalToolCalls: number; totalDurationMs: number; + inputTokens?: number; + outputTokens?: number; + inputCostPerMillion?: number; + outputCostPerMillion?: number; }; } @@ -125,6 +129,7 @@ export interface PullRequestInfo { latestCommit: string; fullName: string; commentId?: number; + labels: string[]; } export interface EnvironmentInfo { @@ -308,6 +313,8 @@ export interface ModelConfig { enabled: boolean; default: boolean; maxTokens: number; + inputCostPerMillion?: number; + outputCostPerMillion?: number; } export interface ProviderConfig { diff --git a/src/shared/types/aiChat.ts b/src/shared/types/aiChat.ts index 97537edc..52dea316 100644 --- a/src/shared/types/aiChat.ts +++ b/src/shared/types/aiChat.ts @@ -130,6 +130,10 @@ export interface DebugMetricsEvent { iterations: number; totalToolCalls: number; totalDurationMs: number; + inputTokens?: number; + outputTokens?: number; + inputCostPerMillion?: number; + outputCostPerMillion?: number; } export type AIChatDebugEvent = DebugContextEvent | DebugToolCallEvent | DebugToolResultEvent | DebugMetricsEvent;