diff --git a/docs/usage.md b/docs/usage.md index f7cc42fcc..fda43de5c 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -80,6 +80,52 @@ navigating to explicit range URLs also pins the range while the bare page URL returns to rolling mode. The `All` preset always pins to `(earliest_session, today)`. +### Model Filter + +The dashboard toolbar includes a **Model** dropdown that scopes +every panel to one or more AI models. By default the button reads +**Model: All** and nothing is filtered. + + + +Open the dropdown for a searchable list of the models found in +your sessions, then click models to include them. The button then +shows the chosen model — for example **Model: gpt-4o** — or +**Model: 3 selected** once several are active. Click the +**All models** row at the top of the list to clear the filter and +return to every model. Selected models also appear as removable +chips beneath the toolbar. + +While a model filter is active, every dashboard panel reflects +only the selected model(s): the summary cards, activity chart, +heatmap, hour-of-week grid, projects, session shape, velocity, +tools, skills, top sessions, and the Session Health rollup. + +Model filtering is **message-grain**, unlike the session-grain +project and agent filters, because a single session can switch +models across turns. A session is included when it has at least +one message from a selected model, and most panels count only the +matching messages. The user turn paired with a matching assistant +turn is kept alongside it — even though a user message carries no +model of its own — so prompts and their responses stay aligned in +the counts and in the top-session evidence. + +**Session Health** is the exception. It is scoped to whole +sessions that used the selected model, but its health scores, +outcomes, tool-failure rates, and compaction counts stay +whole-session aggregates — they are not recomputed from only that +model's messages. + +!!! note "Dashboard-only scope" + The model filter applies only to the analytics dashboard. The + [Session Insights](/insights/) page and the session list are + not scoped by it, so a model selected here does not silently + narrow those views. The [Usage](/token-usage/) page keeps its + own separate model filter. + ### Activity Heatmap A GitHub-style contribution graph showing daily activity. Toggle diff --git a/frontend/src/lib/api/generated/models/DbAnalyticsSummary.ts b/frontend/src/lib/api/generated/models/DbAnalyticsSummary.ts index c91bb7554..7a12ad502 100644 --- a/frontend/src/lib/api/generated/models/DbAnalyticsSummary.ts +++ b/frontend/src/lib/api/generated/models/DbAnalyticsSummary.ts @@ -10,6 +10,7 @@ export type DbAnalyticsSummary = { avg_messages: number; concentration: number; median_messages: number; + models: any[] | null; most_active_project: string; p90_messages: number; token_reporting_sessions: number; diff --git a/frontend/src/lib/api/generated/services/AnalyticsService.test.ts b/frontend/src/lib/api/generated/services/AnalyticsService.test.ts new file mode 100644 index 000000000..e704ba8c0 --- /dev/null +++ b/frontend/src/lib/api/generated/services/AnalyticsService.test.ts @@ -0,0 +1,48 @@ +import { + describe, + expect, + it, + vi, + beforeEach, +} from "vite-plus/test"; + +const { request } = vi.hoisted(() => ({ + request: vi.fn(), +})); + +vi.mock("../core/OpenAPI", () => ({ + OpenAPI: {}, +})); + +vi.mock("../core/request", () => ({ + request, +})); + +import { AnalyticsService } from "./AnalyticsService"; + +describe("AnalyticsService signal sessions", () => { + beforeEach(() => { + request.mockReset(); + request.mockResolvedValue({}); + }); + + it("includes the model filter in signal session requests", async () => { + await AnalyticsService.getApiV1AnalyticsSignalSessions({ + signal: "runaway_tool_loop_count", + from: "2024-06-01", + to: "2024-06-07", + timezone: "UTC", + model: "gpt-4o", + }); + + expect(request).toHaveBeenCalledWith( + {}, + expect.objectContaining({ + query: expect.objectContaining({ + signal: "runaway_tool_loop_count", + model: "gpt-4o", + }), + }), + ); + }); +}); diff --git a/frontend/src/lib/api/generated/services/AnalyticsService.ts b/frontend/src/lib/api/generated/services/AnalyticsService.ts index 4e7868ece..90076aeec 100644 --- a/frontend/src/lib/api/generated/services/AnalyticsService.ts +++ b/frontend/src/lib/api/generated/services/AnalyticsService.ts @@ -30,6 +30,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -64,6 +65,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -111,6 +116,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -148,6 +154,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -182,6 +189,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -229,6 +240,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -266,6 +278,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -299,6 +312,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -342,6 +359,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -378,6 +396,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -411,6 +430,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -454,6 +477,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -490,6 +514,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -523,6 +548,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -566,6 +595,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -603,6 +633,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -641,6 +672,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Filter by model + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -688,6 +723,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -726,6 +762,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -759,6 +796,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -802,6 +843,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -838,6 +880,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -871,6 +914,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -914,6 +961,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -950,6 +998,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -983,6 +1032,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -1026,6 +1079,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -1062,6 +1116,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -1095,6 +1150,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -1138,6 +1197,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -1174,6 +1234,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -1208,6 +1269,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -1255,6 +1320,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, @@ -1292,6 +1358,7 @@ export class AnalyticsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -1325,6 +1392,10 @@ export class AnalyticsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -1368,6 +1439,7 @@ export class AnalyticsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, diff --git a/frontend/src/lib/api/generated/services/TrendsService.ts b/frontend/src/lib/api/generated/services/TrendsService.ts index fd4430e27..47b0cea01 100644 --- a/frontend/src/lib/api/generated/services/TrendsService.ts +++ b/frontend/src/lib/api/generated/services/TrendsService.ts @@ -19,6 +19,7 @@ export class TrendsService { machine, project, agent, + model, dow, hour, minUserMessages, @@ -54,6 +55,10 @@ export class TrendsService { * Filter by agent */ agent?: string, + /** + * Comma-separated model filter + */ + model?: string, /** * Day of week, Monday=0 through Sunday=6 */ @@ -105,6 +110,7 @@ export class TrendsService { 'machine': machine, 'project': project, 'agent': agent, + 'model': model, 'dow': dow, 'hour': hour, 'min_user_messages': minUserMessages, diff --git a/frontend/src/lib/api/types/analytics.ts b/frontend/src/lib/api/types/analytics.ts index 7c902429e..700e6af46 100644 --- a/frontend/src/lib/api/types/analytics.ts +++ b/frontend/src/lib/api/types/analytics.ts @@ -22,6 +22,7 @@ export interface AnalyticsSummary { total_messages: number; total_output_tokens?: number; token_reporting_sessions?: number; + models?: string[]; active_projects: number; active_days: number; avg_messages: number; diff --git a/frontend/src/lib/components/analytics/ActiveFilters.svelte b/frontend/src/lib/components/analytics/ActiveFilters.svelte index 81da8a157..50e589b3e 100644 --- a/frontend/src/lib/components/analytics/ActiveFilters.svelte +++ b/frontend/src/lib/components/analytics/ActiveFilters.svelte @@ -3,6 +3,7 @@ import { CalendarIcon, ClockIcon, + CodeIcon, FolderIcon, MessageSquareTextIcon, MonitorIcon, @@ -16,6 +17,11 @@ ? analytics.agent.split(",") : [], ); + const selectedModels = $derived( + analytics.model + ? analytics.model.split(",").filter((model) => model.length > 0) + : [], + ); const selectedMachines = $derived( analytics.machine ? analytics.machine.split(",") @@ -78,6 +84,7 @@ (analytics.project !== "" ? 1 : 0) + selectedMachines.length + selectedAgents.length + + selectedModels.length + selectedStatuses.length + (analytics.minUserMessages > 0 ? 1 : 0) + (!analytics.includeOneShot ? 1 : 0) + @@ -156,6 +163,20 @@ {/each} + {#each selectedModels as model (model)} + + {/each} + {#if analytics.minUserMessages > 0} diff --git a/frontend/src/lib/stores/analytics.svelte.ts b/frontend/src/lib/stores/analytics.svelte.ts index 03bc6814d..359d87747 100644 --- a/frontend/src/lib/stores/analytics.svelte.ts +++ b/frontend/src/lib/stores/analytics.svelte.ts @@ -62,6 +62,7 @@ class AnalyticsStore { project: string = $state(""); machine: string = $state(""); agent: string = $state(""); + model: string = $state(""); termination: string = $state(""); minUserMessages: number = $state(0); includeOneShot: boolean = $state(true); @@ -143,6 +144,11 @@ class AnalyticsStore { }; private fetchAllVersion = 0; private abortControllers: Partial> = {}; + // Scope key of the cached `signals`: the Analytics-only filters (model plus + // the heatmap drill-down) the cached data was fetched with. Used to drop the + // cache when a fetch crosses the Analytics / Insights boundary, where those + // filters do not exist. + private signalsScope: string | null = null; get timezone(): string { return Intl.DateTimeFormat().resolvedOptions().timeZone; @@ -154,6 +160,7 @@ class AnalyticsStore { this.project !== "" || this.machine !== "" || this.agent !== "" || + this.model !== "" || this.termination !== "" || this.minUserMessages > 0 || !this.includeOneShot || @@ -184,6 +191,7 @@ class AnalyticsStore { this.project = ""; this.machine = ""; this.agent = ""; + this.model = ""; this.termination = ""; this.minUserMessages = 0; this.includeOneShot = true; @@ -214,6 +222,11 @@ class AnalyticsStore { this.fetchAll(); } + clearModel() { + this.model = ""; + this.fetchAll(); + } + toggleAgent(agent: string) { const current = this.agent ? this.agent.split(",") : []; const idx = current.indexOf(agent); @@ -229,6 +242,19 @@ class AnalyticsStore { this.fetchAll(); } + toggleModel(model: string) { + const current = new Set( + this.model.split(",").filter((value) => value.length > 0), + ); + if (current.has(model)) { + current.delete(model); + } else { + current.add(model); + } + this.model = [...current].join(","); + this.fetchAll(); + } + clearMinUserMessages() { this.minUserMessages = 0; sessions.filters.minUserMessages = 0; @@ -348,10 +374,12 @@ class AnalyticsStore { opts: { includeProject?: boolean; includeTime?: boolean; + includeModel?: boolean; } = {}, ): AnalyticsParams { const includeProject = opts.includeProject ?? true; const includeTime = opts.includeTime ?? true; + const includeModel = opts.includeModel ?? true; const p: AnalyticsParams = { from: this.from, to: this.to, @@ -362,6 +390,7 @@ class AnalyticsStore { } if (this.machine) p.machine = this.machine; if (this.agent) p.agent = this.agent; + if (includeModel && this.model) p.model = this.model; if (this.termination) p.termination = this.termination; if (this.minUserMessages > 0) { p.minUserMessages = this.minUserMessages; @@ -388,10 +417,12 @@ class AnalyticsStore { opts: { includeProject?: boolean; includeTime?: boolean; + includeModel?: boolean; } = {}, ): AnalyticsParams { const includeProject = opts.includeProject ?? true; const includeTime = opts.includeTime ?? true; + const includeModel = opts.includeModel ?? true; if (this.selectedDate) { const p: AnalyticsParams = { from: this.selectedDate, @@ -403,6 +434,7 @@ class AnalyticsStore { } if (this.machine) p.machine = this.machine; if (this.agent) p.agent = this.agent; + if (includeModel && this.model) p.model = this.model; if (this.termination) p.termination = this.termination; if (this.minUserMessages > 0) { p.minUserMessages = this.minUserMessages; @@ -426,11 +458,14 @@ class AnalyticsStore { } return p; } - return this.baseParams({ includeProject, includeTime }); + return this.baseParams({ includeProject, includeTime, includeModel }); } signalEvidenceParams(): AnalyticsParams { - return this.filterParams(); + // Insights-only drilldown: omit the Analytics model filter so signal + // evidence matches the unscoped Insights signal facts (the Insights page + // has no model control). + return this.filterParams({ includeModel: false }); } private async executeFetch( @@ -694,12 +729,32 @@ class AnalyticsStore { ); } - async fetchSignals(): Promise { + async fetchSignals( + opts: { includeModel?: boolean } = {}, + ): Promise { + const includeModel = opts.includeModel ?? true; + // `signals` is a cache shared by the Analytics page and the Insights page. + // Key it by the filters that exist on Analytics but not Insights: the model + // and the heatmap drill-down (date/day/hour), which fetchSignalsForInsights + // clears. When this fetch's scope differs from the cached one, drop the + // cache so another scope's signals are never shown while the fetch is in + // flight or retained if it fails; a matching scope keeps the in-place + // refetch behavior used for filter changes shared by both pages. + const scope = JSON.stringify([ + includeModel ? this.model : "", + this.selectedDate, + this.selectedDow, + this.selectedHour, + ]); + if (this.signals !== null && this.signalsScope !== scope) { + this.signals = null; + } + this.signalsScope = scope; return await this.executeFetch( "signals", () => AnalyticsService.getApiV1AnalyticsSignals( - this.filterParams(), + this.filterParams({ includeModel }), ) as unknown as Promise, (data) => { this.signals = data; @@ -713,7 +768,10 @@ class AnalyticsStore { this.selectedDate = null; this.selectedDow = null; this.selectedHour = null; - await this.fetchSignals(); + // The Insights page has no model control and the model filter is an + // Analytics-only scope; omit it so a model selected on Analytics does not + // silently narrow the Insights signal facts. + await this.fetchSignals({ includeModel: false }); } setTopMetric(m: TopSessionsMetric) { diff --git a/frontend/src/lib/stores/analytics.test.ts b/frontend/src/lib/stores/analytics.test.ts index 64e62a24e..62332b418 100644 --- a/frontend/src/lib/stores/analytics.test.ts +++ b/frontend/src/lib/stores/analytics.test.ts @@ -20,6 +20,7 @@ import type { ToolsAnalyticsResponse, SkillsAnalyticsResponse, TopSessionsResponse, + SignalsAnalyticsResponse, } from "../api/types.js"; vi.mock("../api/runtime.js", () => ({ @@ -66,6 +67,7 @@ function makeSummary(): AnalyticsSummary { total_messages: 100, total_output_tokens: 42000, token_reporting_sessions: 8, + models: [], active_projects: 3, active_days: 5, avg_messages: 10, @@ -143,38 +145,8 @@ function makeTopSessions(): TopSessionsResponse { return { metric: "messages", sessions: [] }; } -function mockAllAPIs() { - vi.mocked(analyticsService.getApiV1AnalyticsSummary).mockResolvedValue( - makeSummary(), - ); - vi.mocked(analyticsService.getApiV1AnalyticsActivity).mockResolvedValue( - makeActivity(), - ); - vi.mocked(analyticsService.getApiV1AnalyticsHeatmap).mockResolvedValue( - makeHeatmap(), - ); - vi.mocked(analyticsService.getApiV1AnalyticsProjects).mockResolvedValue( - makeProjects(), - ); - vi.mocked(analyticsService.getApiV1AnalyticsHourOfWeek).mockResolvedValue( - makeHourOfWeek(), - ); - vi.mocked(analyticsService.getApiV1AnalyticsSessions).mockResolvedValue( - makeSessionShape(), - ); - vi.mocked(analyticsService.getApiV1AnalyticsVelocity).mockResolvedValue( - makeVelocity(), - ); - vi.mocked(analyticsService.getApiV1AnalyticsTools).mockResolvedValue( - makeTools(), - ); - vi.mocked(analyticsService.getApiV1AnalyticsSkills).mockResolvedValue( - makeSkills(), - ); - vi.mocked(analyticsService.getApiV1AnalyticsTopSessions).mockResolvedValue( - makeTopSessions(), - ); - vi.mocked(analyticsService.getApiV1AnalyticsSignals).mockResolvedValue({ +function makeSignals(): SignalsAnalyticsResponse { + return { scored_sessions: 0, unscored_sessions: 0, grade_distribution: {}, @@ -224,7 +196,43 @@ function mockAllAPIs() { by_agent: [], by_project: [], calibration: {}, - }); + }; +} + +function mockAllAPIs() { + vi.mocked(analyticsService.getApiV1AnalyticsSummary).mockResolvedValue( + makeSummary(), + ); + vi.mocked(analyticsService.getApiV1AnalyticsActivity).mockResolvedValue( + makeActivity(), + ); + vi.mocked(analyticsService.getApiV1AnalyticsHeatmap).mockResolvedValue( + makeHeatmap(), + ); + vi.mocked(analyticsService.getApiV1AnalyticsProjects).mockResolvedValue( + makeProjects(), + ); + vi.mocked(analyticsService.getApiV1AnalyticsHourOfWeek).mockResolvedValue( + makeHourOfWeek(), + ); + vi.mocked(analyticsService.getApiV1AnalyticsSessions).mockResolvedValue( + makeSessionShape(), + ); + vi.mocked(analyticsService.getApiV1AnalyticsVelocity).mockResolvedValue( + makeVelocity(), + ); + vi.mocked(analyticsService.getApiV1AnalyticsTools).mockResolvedValue( + makeTools(), + ); + vi.mocked(analyticsService.getApiV1AnalyticsSkills).mockResolvedValue( + makeSkills(), + ); + vi.mocked(analyticsService.getApiV1AnalyticsTopSessions).mockResolvedValue( + makeTopSessions(), + ); + vi.mocked(analyticsService.getApiV1AnalyticsSignals).mockResolvedValue( + makeSignals(), + ); } async function loadAnalyticsStore() { @@ -243,6 +251,7 @@ function resetStore() { analytics.agent = ""; analytics.includeAutomated = false; analytics.automatedScope = "human"; + analytics.model = ""; analytics.from = "2024-01-01"; analytics.to = "2024-01-31"; analytics.isPinned = false; @@ -655,6 +664,132 @@ describe("AnalyticsStore automated scope params", () => { }); }); +describe("AnalyticsStore model filter", () => { + it.each([ + { name: "summary", fn: () => analyticsService.getApiV1AnalyticsSummary }, + { name: "activity", fn: () => analyticsService.getApiV1AnalyticsActivity }, + { name: "heatmap", fn: () => analyticsService.getApiV1AnalyticsHeatmap }, + { name: "projects", fn: () => analyticsService.getApiV1AnalyticsProjects }, + { name: "hourOfWeek", fn: () => analyticsService.getApiV1AnalyticsHourOfWeek }, + { name: "sessionShape", fn: () => analyticsService.getApiV1AnalyticsSessions }, + { name: "velocity", fn: () => analyticsService.getApiV1AnalyticsVelocity }, + { name: "tools", fn: () => analyticsService.getApiV1AnalyticsTools }, + { name: "skills", fn: () => analyticsService.getApiV1AnalyticsSkills }, + { name: "topSessions", fn: () => analyticsService.getApiV1AnalyticsTopSessions }, + { name: "signals", fn: () => analyticsService.getApiV1AnalyticsSignals }, + ])("should include model in $name params", ({ fn }) => { + analytics.toggleModel("gpt-4o"); + + const mock = vi.mocked(fn()); + expect(mock).toHaveBeenCalled(); + const params = mock.mock.lastCall?.[0]; + expect(params?.model).toBe("gpt-4o"); + }); + + it("should clear model from subsequent requests", () => { + analytics.toggleModel("gpt-4o"); + vi.clearAllMocks(); + + analytics.clearModel(); + + expect(analytics.model).toBe(""); + const params = + vi.mocked(analyticsService.getApiV1AnalyticsSummary).mock.lastCall?.[0]; + expect(params?.model).toBeUndefined(); + }); + + it("fetchSignalsForInsights omits the model filter", async () => { + analytics.model = "gpt-4o"; + vi.clearAllMocks(); + + await analytics.fetchSignalsForInsights(); + + expect(analyticsService.getApiV1AnalyticsSignals).toHaveBeenCalledWith( + expect.not.objectContaining({ model: expect.anything() }), + ); + // The Insights page has no model control; the selected model stays put for + // the Analytics page rather than being cleared by viewing Insights. + expect(analytics.model).toBe("gpt-4o"); + }); + + it("signalEvidenceParams omits the model filter", () => { + analytics.model = "gpt-4o"; + expect(analytics.signalEvidenceParams().model).toBeUndefined(); + }); + + it("drops stale model-scoped signals while an Insights fetch is pending", async () => { + // Analytics loads model-scoped signals into the shared cache. + analytics.model = "gpt-4o"; + await analytics.fetchSignals(); + expect(analytics.signals).not.toBeNull(); + + // The unmodelled Insights fetch is held in flight. + let resolve!: (v: SignalsAnalyticsResponse) => void; + vi.mocked(analyticsService.getApiV1AnalyticsSignals).mockReturnValue( + new Promise((r) => { + resolve = r; + }), + ); + + const pending = analytics.fetchSignalsForInsights(); + + // The model-scoped cache is dropped up front, so Insights shows a loading + // skeleton instead of another scope's signals during the fetch. + expect(analytics.signals).toBeNull(); + expect(analytics.loading.signals).toBe(true); + + resolve(makeSignals()); + await pending; + expect(analytics.signals).not.toBeNull(); + }); + + it("does not retain stale model-scoped signals when an Insights fetch fails", async () => { + // Analytics loads model-scoped signals into the shared cache. + analytics.model = "gpt-4o"; + await analytics.fetchSignals(); + expect(analytics.signals).not.toBeNull(); + + // The unmodelled Insights fetch fails. + vi.mocked(analyticsService.getApiV1AnalyticsSignals).mockRejectedValueOnce( + new Error("signals failed"), + ); + + await analytics.fetchSignalsForInsights(); + + // The wrong-scope cache is cleared and the failure surfaces rather than + // being swallowed as a cached refetch that keeps stale data. + expect(analytics.signals).toBeNull(); + expect(analytics.errors.signals).toBe("signals failed"); + }); + + it("drops stale drill-down-scoped signals when entering Insights without a model", async () => { + // Analytics loaded signals under a heatmap drill-down (hour) and no model, + // so the cache scope differs from Insights only by the drill-down filter. + analytics.selectedHour = 9; + await analytics.fetchSignals(); + expect(analytics.signals).not.toBeNull(); + + // Insights clears the drill-down; hold the unmodelled fetch in flight. + let resolve!: (v: SignalsAnalyticsResponse) => void; + vi.mocked(analyticsService.getApiV1AnalyticsSignals).mockReturnValue( + new Promise((r) => { + resolve = r; + }), + ); + + const pending = analytics.fetchSignalsForInsights(); + + // The drill-down-scoped cache is dropped even though no model was set, so + // Insights does not show another scope's signals while the fetch runs. + expect(analytics.signals).toBeNull(); + expect(analytics.loading.signals).toBe(true); + + resolve(makeSignals()); + await pending; + expect(analytics.signals).not.toBeNull(); + }); +}); + describe("executeFetch concurrency and error handling", () => { it("should set loading true during fetch", async () => { let resolve!: (v: AnalyticsSummary) => void; diff --git a/internal/db/activityreport.go b/internal/db/activityreport.go index 93219f71c..e304f17ea 100644 --- a/internal/db/activityreport.go +++ b/internal/db/activityreport.go @@ -87,7 +87,7 @@ func (db *DB) GetActivityReport( func (db *DB) activityReportSessions( ctx context.Context, f AnalyticsFilter, rangeStartUTC, rangeEndUTC string, ) ([]activity.SessionMeta, []string, error) { - where, args := f.buildWhereWithDate("", false) + where, args := f.buildWhereWithDate("", false, "s.id") args = append(args, rangeStartUTC, rangeEndUTC) // Each Title candidate is NULLIF'd independently (not a nested diff --git a/internal/db/analytics.go b/internal/db/analytics.go index db3118c35..f2e6a16e2 100644 --- a/internal/db/analytics.go +++ b/internal/db/analytics.go @@ -92,6 +92,7 @@ type AnalyticsFilter struct { Machine string // optional machine filter Project string // optional project filter Agent string // optional agent filter + Model string // optional model filter Timezone string // IANA timezone for day bucketing DayOfWeek *int // nil = all, 0=Mon, 6=Sun (ISO) Hour *int // nil = all, 0-23 @@ -213,7 +214,7 @@ func (f AnalyticsFilter) utcRange() (string, string) { func (f AnalyticsFilter) buildWhere( dateCol string, ) (string, []any) { - return f.buildWhereWithDate(dateCol, true) + return f.buildWhereWithDate(dateCol, true, "sessions.id") } // buildWhereWithoutDate returns common analytics predicates @@ -221,7 +222,7 @@ func (f AnalyticsFilter) buildWhere( // date windows against message timestamps should use this to // avoid pre-filtering by the parent session timestamp. func (f AnalyticsFilter) buildWhereWithoutDate() (string, []any) { - return f.buildWhereWithDate("", false) + return f.buildWhereWithDate("", false, "sessions.id") } func csvFilterValues(raw string) []string { @@ -236,10 +237,82 @@ func csvFilterValues(raw string) []string { return out } +func sqliteAnalyticsCSVPredicate( + col string, + raw string, +) (string, []any) { + values := csvFilterValues(raw) + if len(values) == 0 { + return "", nil + } + if len(values) == 1 { + return col + " = ?", []any{values[0]} + } + placeholders := make([]string, len(values)) + args := make([]any, 0, len(values)) + for i, value := range values { + placeholders[i] = "?" + args = append(args, value) + } + return col + " IN (" + strings.Join(placeholders, ",") + ")", args +} + +func (db *DB) getAnalyticsFilteredMessageCounts( + ctx context.Context, + sessionIDs []string, + f AnalyticsFilter, +) (map[string]int, error) { + stats, err := db.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return nil, err + } + counts := make(map[string]int, len(stats)) + for sessionID, stat := range stats { + counts[sessionID] = stat.Messages + } + return counts, nil +} + +func (db *DB) getAnalyticsModelScopedMessages( + ctx context.Context, + sessionIDs []string, + f AnalyticsFilter, +) (map[string][]ScopedMessage, error) { + scope, err := db.resolveAnalyticsMessageScope(ctx, sessionIDs, f, true) + if err != nil { + return nil, err + } + if scope == nil { + return map[string][]ScopedMessage{}, nil + } + return scope.MessagesBySession(), nil +} + +func (db *DB) getAnalyticsFilteredMessageStats( + ctx context.Context, + sessionIDs []string, + f AnalyticsFilter, +) (map[string]MessageStats, error) { + scope, err := db.resolveAnalyticsMessageScope(ctx, sessionIDs, f, false) + if err != nil { + return nil, err + } + if scope == nil { + return map[string]MessageStats{}, nil + } + return scope.StatsBySession(), nil +} + func (f AnalyticsFilter) buildWhereWithDate( dateCol string, includeDate bool, + sessionIDExpr string, ) (string, []any) { + if sessionIDExpr == "" { + sessionIDExpr = "sessions.id" + } preds := []string{ "message_count > 0", f.RelationshipExclusionSQL(), @@ -302,6 +375,31 @@ func (f AnalyticsFilter) buildWhereWithDate( } } + if f.Model != "" { + models := csvFilterValues(f.Model) + if len(models) == 1 { + preds = append(preds, + "EXISTS (SELECT 1 FROM messages m WHERE "+ + "m.session_id = "+sessionIDExpr+" AND "+ + "m.model = ?)") + args = append(args, models[0]) + } else if len(models) > 1 { + placeholders := make( + []string, len(models), + ) + for i, m := range models { + placeholders[i] = "?" + args = append(args, m) + } + preds = append(preds, + "EXISTS (SELECT 1 FROM messages m WHERE "+ + "m.session_id = "+sessionIDExpr+" AND "+ + "m.model IN ("+ + strings.Join(placeholders, ",")+ + "))") + } + } + if f.MinUserMessages > 0 { preds = append(preds, "user_message_count >= ?") args = append(args, f.MinUserMessages) @@ -360,6 +458,183 @@ func automatedScopePredicate(scope, col string) string { } } +func (db *DB) queryAnalyticsModels( + ctx context.Context, + query string, + args []any, +) ([]string, error) { + rows, err := db.getReader().QueryContext(ctx, ` + `+query, args...) + if err != nil { + return nil, fmt.Errorf("querying analytics models: %w", err) + } + defer rows.Close() + + models := make([]string, 0) + for rows.Next() { + var model string + if err := rows.Scan(&model); err != nil { + return nil, fmt.Errorf("scanning analytics model: %w", err) + } + models = append(models, model) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating analytics models: %w", err) + } + return models, nil +} + +func (db *DB) getAnalyticsModelsSQLiteSummary( + ctx context.Context, + f AnalyticsFilter, + dateCol string, +) ([]string, error) { + if f.HasTimeFilter() { + ids, err := db.filteredSessionIDs(ctx, f) + if err != nil { + return nil, err + } + sessionIDs := make([]string, 0, len(ids)) + for id := range ids { + sessionIDs = append(sessionIDs, id) + } + return db.getAnalyticsModelsForSessionIDsFiltered( + ctx, sessionIDs, f, + ) + } + + where, args := sqliteAnalyticsWhereSQL(f, dateCol, "s.id", true) + return db.queryAnalyticsModels(ctx, ` + SELECT DISTINCT m.model + FROM sessions s + JOIN messages m ON m.session_id = s.id + WHERE `+where+` + AND COALESCE(m.model, '') <> '' + ORDER BY m.model`, + args, + ) +} + +func (db *DB) getAnalyticsModelsForSessionIDs( + ctx context.Context, + sessionIDs []string, +) ([]string, error) { + if len(sessionIDs) == 0 { + return []string{}, nil + } + seen := make(map[string]struct{}, len(sessionIDs)) + unique := make([]string, 0, len(sessionIDs)) + for _, sessionID := range sessionIDs { + if _, ok := seen[sessionID]; ok { + continue + } + seen[sessionID] = struct{}{} + unique = append(unique, sessionID) + } + + modelSet := make(map[string]struct{}) + models := make([]string, 0) + if err := queryChunked(unique, func(chunk []string) error { + ph, args := inPlaceholders(chunk) + found, err := db.queryAnalyticsModels(ctx, ` + SELECT DISTINCT model + FROM messages + WHERE session_id IN `+ph+` + AND COALESCE(model, '') <> '' + ORDER BY model`, + args, + ) + if err != nil { + return err + } + for _, model := range found { + if _, ok := modelSet[model]; ok { + continue + } + modelSet[model] = struct{}{} + models = append(models, model) + } + return nil + }); err != nil { + return nil, err + } + sort.Strings(models) + return models, nil +} + +func (db *DB) getAnalyticsModelsForSessionIDsFiltered( + ctx context.Context, + sessionIDs []string, + f AnalyticsFilter, +) ([]string, error) { + if len(sessionIDs) == 0 { + return []string{}, nil + } + seen := make(map[string]struct{}, len(sessionIDs)) + unique := make([]string, 0, len(sessionIDs)) + for _, sessionID := range sessionIDs { + if _, ok := seen[sessionID]; ok { + continue + } + seen[sessionID] = struct{}{} + unique = append(unique, sessionID) + } + + filterModels := csvFilterValues(f.Model) + allowedModels := make(map[string]struct{}, len(filterModels)) + for _, model := range filterModels { + allowedModels[model] = struct{}{} + } + loc := f.location() + modelSet := make(map[string]struct{}) + models := make([]string, 0) + if err := queryChunked(unique, func(chunk []string) error { + ph, args := inPlaceholders(chunk) + rows, err := db.getReader().QueryContext(ctx, ` + SELECT model, COALESCE(timestamp, '') + FROM messages + WHERE session_id IN `+ph+` + AND COALESCE(model, '') <> ''`, + args..., + ) + if err != nil { + return fmt.Errorf("querying filtered analytics models: %w", err) + } + defer rows.Close() + + for rows.Next() { + var model, ts string + if err := rows.Scan(&model, &ts); err != nil { + return fmt.Errorf("scanning filtered analytics model: %w", err) + } + if len(allowedModels) > 0 { + if _, ok := allowedModels[model]; !ok { + continue + } + } + if f.HasTimeFilter() { + t, ok := localTime(ts, loc) + if !ok || !f.matchesTimeFilter(t) { + continue + } + } + if _, ok := modelSet[model]; ok { + continue + } + modelSet[model] = struct{}{} + models = append(models, model) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating filtered analytics models: %w", err) + } + return nil + }); err != nil { + return nil, err + } + sort.Strings(models) + return models, nil +} + // HasTimeFilter returns true when hour-of-day or day-of-week // filtering is active. func (f AnalyticsFilter) HasTimeFilter() bool { @@ -455,7 +730,9 @@ func sqliteAnalyticsWhereSQL( sessionIDExpr string, includeTime bool, ) (string, []any) { - where, args := f.buildWhere(dateCol) + where, args := f.buildWhereWithDate( + dateCol, true, sessionIDExpr, + ) modifier, _ := f.sqliteTimeModifier() dateExpr := sqliteDateExpr(dateCol, modifier) if f.From != "" { @@ -471,6 +748,12 @@ func sqliteAnalyticsWhereSQL( "m.session_id = " + sessionIDExpr, "m.timestamp != ''", } + if modelPred, modelArgs := sqliteAnalyticsCSVPredicate( + "m.model", f.Model, + ); modelPred != "" { + preds = append(preds, modelPred) + args = append(args, modelArgs...) + } if f.DayOfWeek != nil { dowExpr := "strftime('%w', m.timestamp)" if modifier != "" { @@ -498,13 +781,21 @@ func sqliteAnalyticsWhereSQL( // filteredSessionIDs returns the set of session IDs that have // at least one message matching the hour/dow filter. Used by // session-level queries to restrict results when time filters -// are active. +// are active. With a model filter active it pairs through the +// shared scope reducer (see filteredSessionIDsModel) so an +// empty-model user turn at the selected hour keeps its session, +// matching how the model-scoped panels count it. func (db *DB) filteredSessionIDs( ctx context.Context, f AnalyticsFilter, ) (map[string]bool, error) { + if strings.TrimSpace(f.Model) != "" { + return db.filteredSessionIDsModel(ctx, f) + } loc := f.location() dateCol := "COALESCE(NULLIF(s.started_at, ''), s.created_at)" - where, args := f.buildWhere(dateCol) + where, args := f.buildWhereWithDate( + dateCol, true, "s.id", + ) query := `SELECT s.id, m.timestamp FROM sessions s @@ -546,6 +837,31 @@ func (db *DB) filteredSessionIDs( return ids, nil } +// filteredSessionIDsModel returns the sessions that have at least one +// model-scoped message matching the hour/dow filter. Unlike a direct m.model +// predicate, it runs the shared scope reducer (with the day/hour filter), so an +// empty-model user turn paired with a selected-model assistant keeps its +// session when the user turn falls in the selected hour. +func (db *DB) filteredSessionIDsModel( + ctx context.Context, f AnalyticsFilter, +) (map[string]bool, error) { + sessionIDs, err := db.analyticsModelCandidateSessionIDs(ctx, f) + if err != nil { + return nil, err + } + scope, err := db.resolveAnalyticsMessageScope(ctx, sessionIDs, f, false) + if err != nil { + return nil, err + } + ids := make(map[string]bool) + if scope != nil { + for id := range scope.MessagesBySession() { + ids[id] = true + } + } + return ids, nil +} + // localTime parses a UTC timestamp string and converts it to the // given location. Returns the local time and true on success. func localTime( @@ -628,6 +944,7 @@ type AnalyticsSummary struct { TotalMessages int `json:"total_messages"` TotalOutputTokens int `json:"total_output_tokens"` TokenReportingSessions int `json:"token_reporting_sessions"` + Models []string `json:"models"` ActiveProjects int `json:"active_projects"` ActiveDays int `json:"active_days"` AvgMessages float64 `json:"avg_messages"` @@ -645,7 +962,7 @@ func (db *DB) GetAnalyticsSummary( // The summary is a token/session aggregate, so subagent sessions // (including workflow subagents) are counted here. f.IncludeSubagents = true - if !f.canUseSQLiteTimeSQL() { + if !f.canUseSQLiteTimeSQL() || strings.TrimSpace(f.Model) != "" { return db.getAnalyticsSummaryGo(ctx, f) } dateCol := "COALESCE(NULLIF(started_at, ''), created_at)" @@ -718,7 +1035,10 @@ func (db *DB) GetAnalyticsSummary( return AnalyticsSummary{}, fmt.Errorf("querying analytics summary: %w", err) } - s := AnalyticsSummary{Agents: make(map[string]*AgentSummary)} + s := AnalyticsSummary{ + Agents: make(map[string]*AgentSummary), + Models: []string{}, + } if !rows.Next() { rows.Close() return s, nil @@ -750,6 +1070,12 @@ func (db *DB) GetAnalyticsSummary( fmt.Errorf("closing summary rows: %w", err) } + models, err := db.getAnalyticsModelsSQLiteSummary(ctx, f, dateCol) + if err != nil { + return AnalyticsSummary{}, err + } + s.Models = models + agentRows, err := db.getReader().QueryContext(ctx, ` WITH filtered AS ( SELECT agent, message_count @@ -815,6 +1141,7 @@ func (db *DB) getAnalyticsSummaryGo( defer rows.Close() type sessionRow struct { + id string date string messages int agent string @@ -845,6 +1172,7 @@ func (db *DB) getAnalyticsSummaryGo( continue } all = append(all, sessionRow{ + id: id, date: date, messages: mc, agent: agent, @@ -860,14 +1188,33 @@ func (db *DB) getAnalyticsSummaryGo( var s AnalyticsSummary s.Agents = make(map[string]*AgentSummary) + s.Models = []string{} if len(all) == 0 { return s, nil } + if f.Model != "" { + ids := make([]string, 0, len(all)) + for _, r := range all { + ids = append(ids, r.id) + } + stats, err := db.getAnalyticsFilteredMessageStats(ctx, ids, f) + if err != nil { + return AnalyticsSummary{}, err + } + for i := range all { + stat := stats[all[i].id] + all[i].messages = stat.Messages + all[i].outputTokens = stat.OutputTokens + all[i].hasTokens = stat.HasOutputTokens + } + } + days := make(map[string]bool) projects := make(map[string]int) // project -> message count msgCounts := make([]int, 0, len(all)) + sessionIDs := make([]string, 0, len(all)) for _, r := range all { s.TotalSessions++ @@ -879,6 +1226,7 @@ func (db *DB) getAnalyticsSummaryGo( days[r.date] = true projects[r.project] += r.messages msgCounts = append(msgCounts, r.messages) + sessionIDs = append(sessionIDs, r.id) if s.Agents[r.agent] == nil { s.Agents[r.agent] = &AgentSummary{} @@ -887,6 +1235,22 @@ func (db *DB) getAnalyticsSummaryGo( s.Agents[r.agent].Messages += r.messages } + var models []string + var modelErr error + if strings.TrimSpace(f.Model) != "" || f.HasTimeFilter() { + models, modelErr = db.getAnalyticsModelsForSessionIDsFiltered( + ctx, sessionIDs, f, + ) + } else { + models, modelErr = db.getAnalyticsModelsForSessionIDs( + ctx, sessionIDs, + ) + } + if modelErr != nil { + return s, modelErr + } + s.Models = models + s.ActiveProjects = len(projects) s.ActiveDays = len(days) s.AvgMessages = math.Round( @@ -977,6 +1341,154 @@ func bucketDate(date string, granularity string) string { } } +func (db *DB) getModelScopedToolCallCounts( + ctx context.Context, + sessionIDs []string, + f AnalyticsFilter, +) (map[string]int, error) { + counts := make(map[string]int, len(sessionIDs)) + if len(sessionIDs) == 0 || strings.TrimSpace(f.Model) == "" { + return counts, nil + } + flt := f.messageScopeFilter() + loc := f.location() + if err := queryChunked(sessionIDs, func(chunk []string) error { + ph, args := inPlaceholders(chunk) + rows, err := db.getReader().QueryContext(ctx, ` + SELECT tc.session_id, m.model, COALESCE(m.timestamp, ''), COUNT(*) + FROM tool_calls tc + JOIN messages m + ON m.session_id = tc.session_id + AND m.id = tc.message_id + WHERE tc.session_id IN `+ph+` + GROUP BY tc.session_id, m.model, COALESCE(m.timestamp, '')`, + args..., + ) + if err != nil { + return fmt.Errorf("querying model-scoped analytics tool calls: %w", err) + } + defer rows.Close() + for rows.Next() { + var sessionID, model, ts string + var count int + if err := rows.Scan(&sessionID, &model, &ts, &count); err != nil { + return fmt.Errorf("scanning model-scoped analytics tool calls: %w", err) + } + if _, ok := flt.Models[model]; !ok { + continue + } + parsed, has := localTime(ts, loc) + if !flt.MatchesDayHour(parsed, has) { + continue + } + counts[sessionID] += count + } + return rows.Err() + }); err != nil { + return nil, err + } + return counts, nil +} + +func (db *DB) getAnalyticsActivityFilteredByModelTime( + ctx context.Context, + f AnalyticsFilter, + granularity string, +) (ActivityResponse, error) { + loc := f.location() + dateCol := "COALESCE(NULLIF(started_at, ''), created_at)" + where, args := f.buildWhereWithDate(dateCol, true, "sessions.id") + var timeIDs map[string]bool + if f.HasTimeFilter() { + var err error + timeIDs, err = db.filteredSessionIDs(ctx, f) + if err != nil { + return ActivityResponse{}, err + } + } + + rows, err := db.getReader().QueryContext(ctx, `SELECT id, `+dateCol+`, agent + FROM sessions + WHERE `+where, args...) + if err != nil { + return ActivityResponse{}, + fmt.Errorf("querying analytics activity sessions: %w", err) + } + defer rows.Close() + + type sessionRow struct { + id, date, agent string + } + sessions := make([]sessionRow, 0) + sessionIDs := make([]string, 0) + for rows.Next() { + var id, ts, agent string + if err := rows.Scan(&id, &ts, &agent); err != nil { + return ActivityResponse{}, + fmt.Errorf("scanning analytics activity session: %w", err) + } + date := localDate(ts, loc) + if !inDateRange(date, f.From, f.To) { + continue + } + if timeIDs != nil && !timeIDs[id] { + continue + } + sessions = append(sessions, sessionRow{id: id, date: date, agent: agent}) + sessionIDs = append(sessionIDs, id) + } + if err := rows.Err(); err != nil { + return ActivityResponse{}, + fmt.Errorf("iterating analytics activity sessions: %w", err) + } + + messageStats, err := db.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return ActivityResponse{}, err + } + toolCalls, err := db.getModelScopedToolCallCounts( + ctx, sessionIDs, f, + ) + if err != nil { + return ActivityResponse{}, err + } + + buckets := make(map[string]*ActivityEntry) + for _, session := range sessions { + bucket := bucketDate(session.date, granularity) + entry := buckets[bucket] + if entry == nil { + entry = &ActivityEntry{ + Date: bucket, + ByAgent: make(map[string]int), + } + buckets[bucket] = entry + } + entry.Sessions++ + stat := messageStats[session.id] + entry.Messages += stat.Messages + entry.UserMessages += stat.UserMessages + entry.AssistantMessages += stat.AssistantMessages + entry.ThinkingMessages += stat.ThinkingMessages + entry.ToolCalls += toolCalls[session.id] + entry.ByAgent[session.agent] += stat.Messages + } + + series := make([]ActivityEntry, 0, len(buckets)) + for _, entry := range buckets { + series = append(series, *entry) + } + sort.Slice(series, func(i, j int) bool { + return series[i].Date < series[j].Date + }) + return ActivityResponse{ + Granularity: granularity, + Series: series, + }, nil +} + // GetAnalyticsActivity returns session/message counts grouped // by time bucket. func (db *DB) GetAnalyticsActivity( @@ -986,9 +1498,20 @@ func (db *DB) GetAnalyticsActivity( if granularity == "" { granularity = "day" } + if strings.TrimSpace(f.Model) != "" { + return db.getAnalyticsActivityFilteredByModelTime( + ctx, f, granularity, + ) + } loc := f.location() dateCol := "COALESCE(NULLIF(s.started_at, ''), s.created_at)" - where, args := f.buildWhere(dateCol) + where, args := f.buildWhereWithDate(dateCol, true, "s.id") + if modelPred, modelArgs := sqliteAnalyticsCSVPredicate( + "m.model", f.Model, + ); modelPred != "" { + where += " AND " + modelPred + args = append(args, modelArgs...) + } var timeIDs map[string]bool if f.HasTimeFilter() { @@ -1082,7 +1605,7 @@ func (db *DB) GetAnalyticsActivity( err = queryChunked(sessionIDs, func(chunk []string) error { return db.mergeActivityToolCalls( - ctx, chunk, sessionSeen, buckets, + ctx, chunk, sessionSeen, buckets, f.Model, ) }) if err != nil { @@ -1112,12 +1635,26 @@ func (db *DB) mergeActivityToolCalls( chunk []string, sessionBucket map[string]string, buckets map[string]*ActivityEntry, + model string, ) error { ph, args := inPlaceholders(chunk) - q := `SELECT session_id, COUNT(*) - FROM tool_calls - WHERE session_id IN ` + ph + ` - GROUP BY session_id` + q := `SELECT tc.session_id, COUNT(*) + FROM tool_calls tc` + if model != "" { + q += ` + JOIN messages m + ON m.session_id = tc.session_id AND m.id = tc.message_id` + } + q += ` + WHERE tc.session_id IN ` + ph + if modelPred, modelArgs := sqliteAnalyticsCSVPredicate( + "m.model", model, + ); modelPred != "" { + q += ` AND ` + modelPred + args = append(args, modelArgs...) + } + q += ` + GROUP BY tc.session_id` rows, err := db.getReader().QueryContext(ctx, q, args...) if err != nil { return fmt.Errorf( @@ -1201,7 +1738,16 @@ func (db *DB) GetAnalyticsHeatmap( } defer rows.Close() - dayCounts := make(map[string]int) // date -> count + type heatmapRow struct { + id string + date string + messages int + outputTokens int + hasTokens bool + } + + var heatmapRows []heatmapRow + dayCounts := make(map[string]int) daySessions := make(map[string]int) dayOutputTokens := make(map[string]int) @@ -1222,17 +1768,44 @@ func (db *DB) GetAnalyticsHeatmap( if timeIDs != nil && !timeIDs[id] { continue } - dayCounts[date] += mc - daySessions[date]++ - if hasTokens { - dayOutputTokens[date] += outputTokens - } + heatmapRows = append(heatmapRows, heatmapRow{ + id: id, + date: date, + messages: mc, + outputTokens: outputTokens, + hasTokens: hasTokens, + }) } if err := rows.Err(); err != nil { return HeatmapResponse{}, fmt.Errorf("iterating heatmap rows: %w", err) } + if f.Model != "" && (metric == "messages" || metric == "output_tokens") { + ids := make([]string, 0, len(heatmapRows)) + for _, row := range heatmapRows { + ids = append(ids, row.id) + } + stats, err := db.getAnalyticsFilteredMessageStats(ctx, ids, f) + if err != nil { + return HeatmapResponse{}, err + } + for i := range heatmapRows { + stat := stats[heatmapRows[i].id] + heatmapRows[i].messages = stat.Messages + heatmapRows[i].outputTokens = stat.OutputTokens + heatmapRows[i].hasTokens = stat.HasOutputTokens + } + } + + for _, row := range heatmapRows { + dayCounts[row.date] += row.messages + daySessions[row.date]++ + if row.hasTokens { + dayOutputTokens[row.date] += row.outputTokens + } + } + // Choose which map to use based on metric source := dayCounts switch metric { @@ -1430,6 +2003,14 @@ func (db *DB) GetAnalyticsProjects( projectMap := make(map[string]*projectData) var projectOrder []string + type projectRow struct { + id string + project string + date string + messages int + agent string + } + var projectRows []projectRow for rows.Next() { var id, project, ts, agent string @@ -1447,35 +2028,58 @@ func (db *DB) GetAnalyticsProjects( if timeIDs != nil && !timeIDs[id] { continue } + projectRows = append(projectRows, projectRow{ + id: id, + project: project, + date: date, + messages: mc, + agent: agent, + }) + } + if err := rows.Err(); err != nil { + return ProjectsAnalyticsResponse{}, + fmt.Errorf("iterating project rows: %w", err) + } - pd, ok := projectMap[project] + if f.Model != "" { + ids := make([]string, 0, len(projectRows)) + for _, row := range projectRows { + ids = append(ids, row.id) + } + counts, err := db.getAnalyticsFilteredMessageCounts(ctx, ids, f) + if err != nil { + return ProjectsAnalyticsResponse{}, err + } + for i := range projectRows { + projectRows[i].messages = counts[projectRows[i].id] + } + } + + for _, row := range projectRows { + pd, ok := projectMap[row.project] if !ok { pd = &projectData{ - name: project, + name: row.project, agents: make(map[string]int), days: make(map[string]int), } - projectMap[project] = pd - projectOrder = append(projectOrder, project) + projectMap[row.project] = pd + projectOrder = append(projectOrder, row.project) } pd.sessions++ - pd.messages += mc - pd.counts = append(pd.counts, mc) - pd.agents[agent]++ - pd.days[date] += mc + pd.messages += row.messages + pd.counts = append(pd.counts, row.messages) + pd.agents[row.agent]++ + pd.days[row.date] += row.messages - if pd.first == "" || date < pd.first { - pd.first = date + if pd.first == "" || row.date < pd.first { + pd.first = row.date } - if date > pd.last { - pd.last = date + if row.date > pd.last { + pd.last = row.date } } - if err := rows.Err(); err != nil { - return ProjectsAnalyticsResponse{}, - fmt.Errorf("iterating project rows: %w", err) - } projects := make([]ProjectAnalytics, 0, len(projectMap)) for _, name := range projectOrder { @@ -1541,9 +2145,12 @@ type HourOfWeekResponse struct { func (db *DB) GetAnalyticsHourOfWeek( ctx context.Context, f AnalyticsFilter, ) (HourOfWeekResponse, error) { + if strings.TrimSpace(f.Model) != "" { + return db.getAnalyticsHourOfWeekFilteredByModel(ctx, f) + } loc := f.location() dateCol := "COALESCE(NULLIF(s.started_at, ''), s.created_at)" - where, args := f.buildWhere(dateCol) + where, args := f.buildWhereWithDate(dateCol, true, "s.id") query := `SELECT ` + dateCol + `, m.timestamp FROM sessions s @@ -1582,6 +2189,91 @@ func (db *DB) GetAnalyticsHourOfWeek( fmt.Errorf("iterating hour-of-week rows: %w", err) } + return HourOfWeekResponseFromGrid(grid), nil +} + +// getAnalyticsHourOfWeekFilteredByModel buckets model-scoped messages by +// day-of-week and hour. Like every model-filtered panel it pairs empty-model +// user turns with their selected-model assistant via the shared scope reducer, +// so those user turns appear in the heatmap consistently with the summary, +// activity, velocity, and trends panels. The heatmap is the control that sets +// the day/hour filter, so it clears DayOfWeek/Hour before scoping to keep +// showing the full grid, matching the no-model path. +// analyticsModelCandidateSessionIDs returns the date-filtered, model-scoped +// session IDs that feed the shared message-scope reducer. The day/hour filter +// is intentionally not applied here: callers that need it let the reducer +// apply it (so paired empty-model user turns are kept), while the hour-of-week +// heatmap clears it to show the full grid. +func (db *DB) analyticsModelCandidateSessionIDs( + ctx context.Context, f AnalyticsFilter, +) ([]string, error) { + loc := f.location() + dateCol := "COALESCE(NULLIF(started_at, ''), created_at)" + where, args := f.buildWhereWithDate(dateCol, true, "sessions.id") + + rows, err := db.getReader().QueryContext(ctx, `SELECT id, `+dateCol+` + FROM sessions + WHERE `+where, args...) + if err != nil { + return nil, fmt.Errorf("querying model candidate sessions: %w", err) + } + defer rows.Close() + + ids := make([]string, 0) + for rows.Next() { + var id, ts string + if err := rows.Scan(&id, &ts); err != nil { + return nil, fmt.Errorf("scanning model candidate session: %w", err) + } + if !inDateRange(localDate(ts, loc), f.From, f.To) { + continue + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating model candidate sessions: %w", err) + } + return ids, nil +} + +func (db *DB) getAnalyticsHourOfWeekFilteredByModel( + ctx context.Context, f AnalyticsFilter, +) (HourOfWeekResponse, error) { + sessionIDs, err := db.analyticsModelCandidateSessionIDs(ctx, f) + if err != nil { + return HourOfWeekResponse{}, err + } + + scopeFilter := f + scopeFilter.DayOfWeek = nil + scopeFilter.Hour = nil + scope, err := db.resolveAnalyticsMessageScope( + ctx, sessionIDs, scopeFilter, false, + ) + if err != nil { + return HourOfWeekResponse{}, err + } + + var grid [7][24]int + if scope != nil { + for _, msgs := range scope.MessagesBySession() { + for _, m := range msgs { + if !m.HasLocalTime { + continue + } + // Go Sunday=0, convert to ISO Monday=0 + dow := (int(m.LocalTime.Weekday()) + 6) % 7 + grid[dow][m.LocalTime.Hour()]++ + } + } + } + + return HourOfWeekResponseFromGrid(grid), nil +} + +// HourOfWeekResponseFromGrid flattens the 7x24 grid into the dense 168-cell +// response. +func HourOfWeekResponseFromGrid(grid [7][24]int) HourOfWeekResponse { cells := make([]HourOfWeekCell, 0, 168) for d := range 7 { for h := range 24 { @@ -1592,8 +2284,7 @@ func (db *DB) GetAnalyticsHourOfWeek( }) } } - - return HourOfWeekResponse{Cells: cells}, nil + return HourOfWeekResponse{Cells: cells} } // --- Session Shape --- @@ -1714,6 +2405,7 @@ func (db *DB) GetAnalyticsSessionShape( ctx context.Context, f AnalyticsFilter, ) (SessionShapeResponse, error) { loc := f.location() + modelFilter := strings.TrimSpace(f.Model) != "" dateCol := "COALESCE(NULLIF(started_at, ''), created_at)" where, args := f.buildWhere(dateCol) @@ -1761,7 +2453,9 @@ func (db *DB) GetAnalyticsSessionShape( } totalCount++ - lengthCounts[lengthBucket(mc)]++ + if !modelFilter { + lengthCounts[lengthBucket(mc)]++ + } sessionIDs = append(sessionIDs, id) if startedAt != nil && endedAt != nil && @@ -1781,9 +2475,30 @@ func (db *DB) GetAnalyticsSessionShape( fmt.Errorf("iterating session shape rows: %w", err) } - // Query autonomy data for filtered sessions autonomyCounts := make(map[string]int) - if len(sessionIDs) > 0 { + if modelFilter && len(sessionIDs) > 0 { + stats, err := db.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return SessionShapeResponse{}, err + } + lengthCounts = make(map[string]int) + seen := make(map[string]struct{}, len(sessionIDs)) + for _, sessionID := range sessionIDs { + if _, ok := seen[sessionID]; ok { + continue + } + seen[sessionID] = struct{}{} + stat := stats[sessionID] + lengthCounts[lengthBucket(stat.Messages)]++ + if stat.UserMessages > 0 { + ratio := float64(stat.ToolUseMessages) / + float64(stat.UserMessages) + autonomyCounts[autonomyBucket(ratio)]++ + } + } + } else if len(sessionIDs) > 0 { err := queryChunked(sessionIDs, func(chunk []string) error { return db.queryAutonomyChunk( @@ -2071,23 +2786,55 @@ func skillProjectBreakdowns( return out } -func analyticsToolsQuery(placeholders string) string { - return `SELECT session_id, category, COUNT(*) - FROM tool_calls - WHERE session_id IN ` + placeholders + ` - GROUP BY session_id, category` +func analyticsToolsQuery( + placeholders string, + modelPred string, + includeMessageMeta bool, +) string { + query := `SELECT tc.session_id, tc.category, COUNT(*)` + if includeMessageMeta { + query += `, COALESCE(m.timestamp, '')` + } + query += ` + FROM tool_calls tc` + if includeMessageMeta { + query += ` + LEFT JOIN messages m + ON m.session_id = tc.session_id AND m.id = tc.message_id` + } + query += ` + WHERE tc.session_id IN ` + placeholders + if modelPred != "" { + query += ` + AND ` + modelPred + } + query += ` + GROUP BY tc.session_id, tc.category` + if includeMessageMeta { + query += `, COALESCE(m.timestamp, '')` + } + return query } -func analyticsSkillsQuery(placeholders string) string { - return `SELECT tc.session_id, TRIM(tc.skill_name), COUNT(*), +func analyticsSkillsQuery( + placeholders string, + modelPred string, +) string { + query := `SELECT tc.session_id, TRIM(tc.skill_name), COUNT(*), COALESCE(m.timestamp, '') FROM tool_calls tc LEFT JOIN messages m ON m.session_id = tc.session_id AND m.id = tc.message_id WHERE tc.session_id IN ` + placeholders + ` - AND TRIM(COALESCE(tc.skill_name, '')) != '' + AND TRIM(COALESCE(tc.skill_name, '')) != ''` + if modelPred != "" { + query += ` + AND ` + modelPred + } + query += ` GROUP BY tc.session_id, TRIM(tc.skill_name), COALESCE(m.timestamp, '')` + return query } // GetAnalyticsTools returns tool usage analytics aggregated @@ -2095,18 +2842,8 @@ func analyticsSkillsQuery(placeholders string) string { func (db *DB) GetAnalyticsTools( ctx context.Context, f AnalyticsFilter, ) (ToolsAnalyticsResponse, error) { - loc := f.location() dateCol := "COALESCE(NULLIF(started_at, ''), created_at)" - where, args := f.buildWhere(dateCol) - - var timeIDs map[string]bool - if f.HasTimeFilter() { - var err error - timeIDs, err = db.filteredSessionIDs(ctx, f) - if err != nil { - return ToolsAnalyticsResponse{}, err - } - } + where, args := f.buildWhereWithoutDate() // Fetch filtered session IDs and their metadata. sessQ := `SELECT id, ` + dateCol + `, agent @@ -2120,7 +2857,7 @@ func (db *DB) GetAnalyticsTools( defer sessRows.Close() type sessInfo struct { - date string + ts string agent string } sessionMap := make(map[string]sessInfo) @@ -2132,14 +2869,7 @@ func (db *DB) GetAnalyticsTools( return ToolsAnalyticsResponse{}, fmt.Errorf("scanning tool session: %w", err) } - date := localDate(ts, loc) - if !inDateRange(date, f.From, f.To) { - continue - } - if timeIDs != nil && !timeIDs[id] { - continue - } - sessionMap[id] = sessInfo{date: date, agent: agent} + sessionMap[id] = sessInfo{ts: ts, agent: agent} sessionIDs = append(sessionIDs, id) } if err := sessRows.Err(); err != nil { @@ -2162,13 +2892,18 @@ func (db *DB) GetAnalyticsTools( sessionID string category string count int + date string } var toolRows []toolRow err = queryChunked(sessionIDs, func(chunk []string) error { ph, chunkArgs := inPlaceholders(chunk) - q := analyticsToolsQuery(ph) + modelPred, modelArgs := sqliteAnalyticsCSVPredicate( + "m.model", f.Model, + ) + chunkArgs = append(chunkArgs, modelArgs...) + q := analyticsToolsQuery(ph, modelPred, true) rows, qErr := db.getReader().QueryContext( ctx, q, chunkArgs..., ) @@ -2179,19 +2914,30 @@ func (db *DB) GetAnalyticsTools( } defer rows.Close() for rows.Next() { - var sid, cat string + var sid, cat, ts string var count int if err := rows.Scan( - &sid, &cat, &count, + &sid, &cat, &count, &ts, ); err != nil { return fmt.Errorf( "scanning tool_call: %w", err, ) } + info, ok := sessionMap[sid] + if !ok { + continue + } + _, date, keep := f.ResolveSkillRowTime( + ts, info.ts, + ) + if !keep { + continue + } toolRows = append(toolRows, toolRow{ sessionID: sid, category: cat, count: count, + date: date, }) } return rows.Err() @@ -2218,7 +2964,7 @@ func (db *DB) GetAnalyticsTools( } agentCats[info.agent][tr.category] += tr.count - week := bucketDate(info.date, "week") + week := bucketDate(tr.date, "week") if trendBuckets[week] == nil { trendBuckets[week] = make(map[string]int) } @@ -2389,7 +3135,11 @@ func (db *DB) GetAnalyticsSkills( err = queryChunked(sessionIDs, func(chunk []string) error { ph, chunkArgs := inPlaceholders(chunk) - q := analyticsSkillsQuery(ph) + modelPred, modelArgs := sqliteAnalyticsCSVPredicate( + "m.model", f.Model, + ) + chunkArgs = append(chunkArgs, modelArgs...) + q := analyticsSkillsQuery(ph, modelPred) rows, qErr := db.getReader().QueryContext( ctx, q, chunkArgs..., ) @@ -2497,6 +3247,42 @@ func (db *DB) queryVelocityMsgs( return rows.Err() } +func (db *DB) getAnalyticsVelocityMessages( + ctx context.Context, + sessionIDs []string, + f AnalyticsFilter, +) (map[string][]velocityMsg, error) { + sessionMsgs := make(map[string][]velocityMsg, len(sessionIDs)) + if len(sessionIDs) == 0 { + return sessionMsgs, nil + } + + loc := f.location() + if strings.TrimSpace(f.Model) == "" { + err := queryChunked(sessionIDs, func(chunk []string) error { + return db.queryVelocityMsgs(ctx, chunk, loc, sessionMsgs) + }) + return sessionMsgs, err + } + + scope, err := db.resolveAnalyticsMessageScope(ctx, sessionIDs, f, false) + if err != nil { + return nil, err + } + if scope == nil { + return sessionMsgs, nil + } + for sessionID, rows := range scope.TimingBySession() { + for _, row := range rows { + sessionMsgs[sessionID] = append(sessionMsgs[sessionID], velocityMsg{ + role: row.Role, ts: row.Time, valid: row.Valid, + contentLength: row.ContentLength, + }) + } + } + return sessionMsgs, nil +} + // Percentiles holds p50 and p90 values. type Percentiles struct { P50 float64 `json:"p50"` @@ -2840,53 +3626,70 @@ func (db *DB) GetAnalyticsVelocity( ByComplexity: []VelocityBreakdown{}, }, nil } + if strings.TrimSpace(f.Model) != "" { + stats, err := db.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return VelocityResponse{}, err + } + for _, sid := range sessionIDs { + info := sessionMap[sid] + info.mc = stats[sid].Messages + sessionMap[sid] = info + } + } - // Phase 2: Fetch messages for filtered sessions (chunked) - sessionMsgs := make(map[string][]velocityMsg) - err = queryChunked(sessionIDs, - func(chunk []string) error { - return db.queryVelocityMsgs( - ctx, chunk, loc, sessionMsgs, - ) - }) + sessionMsgs, err := db.getAnalyticsVelocityMessages( + ctx, sessionIDs, f, + ) if err != nil { return VelocityResponse{}, err } - // Phase 2b: Fetch tool call counts per session (chunked) - toolCountMap := make(map[string]int) - err = queryChunked(sessionIDs, - func(chunk []string) error { - ph, chunkArgs := inPlaceholders(chunk) - q := `SELECT session_id, COUNT(*) - FROM tool_calls - WHERE session_id IN ` + ph + ` - GROUP BY session_id` - rows, qErr := db.getReader().QueryContext( - ctx, q, chunkArgs..., - ) - if qErr != nil { - return fmt.Errorf( - "querying velocity tool_calls: %w", - qErr, + var toolCountMap map[string]int + if strings.TrimSpace(f.Model) != "" { + toolCountMap, err = db.getModelScopedToolCallCounts( + ctx, sessionIDs, f, + ) + if err != nil { + return VelocityResponse{}, err + } + } else { + toolCountMap = make(map[string]int) + err = queryChunked(sessionIDs, + func(chunk []string) error { + ph, chunkArgs := inPlaceholders(chunk) + q := `SELECT session_id, COUNT(*) + FROM tool_calls + WHERE session_id IN ` + ph + ` + GROUP BY session_id` + rows, qErr := db.getReader().QueryContext( + ctx, q, chunkArgs..., ) - } - defer rows.Close() - for rows.Next() { - var sid string - var count int - if err := rows.Scan(&sid, &count); err != nil { + if qErr != nil { return fmt.Errorf( - "scanning velocity tool_call: %w", - err, + "querying velocity tool_calls: %w", + qErr, ) } - toolCountMap[sid] = count - } - return rows.Err() - }) - if err != nil { - return VelocityResponse{}, err + defer rows.Close() + for rows.Next() { + var sid string + var count int + if err := rows.Scan(&sid, &count); err != nil { + return fmt.Errorf( + "scanning velocity tool_call: %w", + err, + ) + } + toolCountMap[sid] = count + } + return rows.Err() + }) + if err != nil { + return VelocityResponse{}, err + } } // Process per-session metrics @@ -3141,6 +3944,15 @@ type SignalRow struct { } // GetAnalyticsSignals returns aggregated session signal data. +// +// Signals are session-scoped: the counts come from persisted per-session +// columns computed at parse time (health, context pressure, compaction, +// quality markers) that describe the whole session, not one model's turns. +// When a model filter is active the session set is scoped to sessions that +// used the model, but the totals stay session-level aggregates and are not +// re-attributed per model -- per-model signal attribution is not well defined +// for most signals and is intentionally not computed here. The PostgreSQL and +// DuckDB stores mirror this. func (db *DB) GetAnalyticsSignals( ctx context.Context, f AnalyticsFilter, ) (SignalsAnalyticsResponse, error) { @@ -3231,7 +4043,9 @@ func (db *DB) GetAnalyticsSignals( } // GetAnalyticsSignalSessions returns concrete examples for a -// signal within the current analytics filter. +// signal within the current analytics filter. Candidates are ranked by the +// session-level signal counts (see GetAnalyticsSignals); under a model filter +// the drill-down evidence is model-scoped while the ranking stays session-level. func (db *DB) GetAnalyticsSignalSessions( ctx context.Context, f AnalyticsFilter, @@ -3252,7 +4066,7 @@ func (db *DB) GetAnalyticsSignalSessions( return SignalSessionsResponse{}, err } candidates := SignalCandidates(rows, signal, limit) - messages, err := db.signalMessages(ctx, candidates) + messages, err := db.signalMessages(ctx, candidates, f) if err != nil { return SignalSessionsResponse{}, err } @@ -3442,6 +4256,7 @@ func SignalCandidates( func (db *DB) signalMessages( ctx context.Context, rows []SignalRow, + f AnalyticsFilter, ) (map[string][]SignalMessage, error) { out := make(map[string][]SignalMessage, len(rows)) if len(rows) == 0 { @@ -3451,12 +4266,42 @@ func (db *DB) signalMessages( for _, r := range rows { ids = append(ids, r.ID) } + if strings.TrimSpace(f.Model) != "" { + rowsBySession, err := db.getAnalyticsModelScopedMessages(ctx, ids, f) + if err != nil { + return nil, err + } + for sessionID, scopedRows := range rowsBySession { + for _, row := range scopedRows { + out[sessionID] = append(out[sessionID], SignalMessage{ + SessionID: row.SessionID, + Ordinal: row.Ordinal, + Role: row.Role, + Content: row.Content, + Timestamp: row.Timestamp, + IsSystem: row.IsSystem, + HasToolUse: row.HasToolUse, + }) + } + } + return out, nil + } + filterModels := csvFilterValues(f.Model) err := queryChunked(ids, func(chunk []string) error { ph, args := inPlaceholders(chunk) q := `SELECT session_id, ordinal, role, content, COALESCE(timestamp, ''), is_system, has_tool_use FROM messages - WHERE session_id IN ` + ph + ` + WHERE session_id IN ` + ph + if len(filterModels) == 1 { + q += ` AND model = ?` + args = append(args, filterModels[0]) + } else if len(filterModels) > 1 { + modelPH, modelArgs := inPlaceholders(filterModels) + q += ` AND model IN ` + modelPH + args = append(args, modelArgs...) + } + q += ` ORDER BY session_id, ordinal` msgRows, err := db.getReader().QueryContext(ctx, q, args...) if err != nil { @@ -4459,7 +5304,7 @@ func (db *DB) GetAnalyticsTopSessions( if metric == "" { metric = "messages" } - if !f.canUseSQLiteTimeSQL() { + if !f.canUseSQLiteTimeSQL() || strings.TrimSpace(f.Model) != "" { return db.getAnalyticsTopSessionsGo(ctx, f, metric) } dateCol := "COALESCE(NULLIF(started_at, ''), created_at)" @@ -4474,7 +5319,9 @@ func (db *DB) GetAnalyticsTopSessions( var orderExpr string switch metric { case "output_tokens": - where += " AND has_total_output_tokens = TRUE" + if strings.TrimSpace(f.Model) == "" { + where += " AND has_total_output_tokens = TRUE" + } orderExpr = "total_output_tokens DESC, id ASC" case "duration": orderExpr = activeDurationSelectExpr + " DESC, id ASC" @@ -4560,7 +5407,9 @@ func (db *DB) getAnalyticsTopSessionsGo( } limitClause := " LIMIT 200" - if f.HasTimeFilter() || needsGoSort { + if f.HasTimeFilter() || needsGoSort || + (strings.TrimSpace(f.Model) != "" && + (metric == "messages" || metric == "output_tokens")) { limitClause = "" } @@ -4575,7 +5424,6 @@ func (db *DB) getAnalyticsTopSessionsGo( activeDurationSelectExpr = "COALESCE(" + sqliteActiveDurationExpr("sessions.id") + ", 0)" } - query := `SELECT id, ` + dateCol + `, project, first_message, COALESCE(display_name, session_name) AS display_name, @@ -4643,6 +5491,41 @@ func (db *DB) getAnalyticsTopSessionsGo( fmt.Errorf("iterating top sessions: %w", err) } + if strings.TrimSpace(f.Model) != "" && + (metric == "messages" || metric == "output_tokens") { + sessionIDs := make([]string, 0, len(sessions)) + for _, session := range sessions { + sessionIDs = append(sessionIDs, session.ID) + } + stats, err := db.getAnalyticsFilteredMessageStats(ctx, sessionIDs, f) + if err != nil { + return TopSessionsResponse{}, err + } + filtered := sessions[:0] + for i := range sessions { + stat := stats[sessions[i].ID] + sessions[i].MessageCount = stat.Messages + sessions[i].OutputTokens = stat.OutputTokens + if metric == "output_tokens" && !stat.HasOutputTokens { + continue + } + filtered = append(filtered, sessions[i]) + } + sessions = filtered + sort.SliceStable(sessions, func(i, j int) bool { + if metric == "output_tokens" { + if sessions[i].OutputTokens != sessions[j].OutputTokens { + return sessions[i].OutputTokens > + sessions[j].OutputTokens + } + } else if sessions[i].MessageCount != sessions[j].MessageCount { + return sessions[i].MessageCount > + sessions[j].MessageCount + } + return sessions[i].ID < sessions[j].ID + }) + } + if sessions == nil { sessions = []TopSession{} } diff --git a/internal/db/analytics_scope.go b/internal/db/analytics_scope.go new file mode 100644 index 000000000..484bf620f --- /dev/null +++ b/internal/db/analytics_scope.go @@ -0,0 +1,148 @@ +package db + +import ( + "context" + "fmt" + "strings" +) + +// messageScopeFilter adapts the model/day/hour parts of an AnalyticsFilter into +// the pure ScopeFilter. +func (f AnalyticsFilter) messageScopeFilter() ScopeFilter { + models := make(map[string]struct{}) + for _, m := range csvFilterValues(f.Model) { + models[m] = struct{}{} + } + return ScopeFilter{ + Models: models, + DayOfWeek: f.DayOfWeek, + Hour: f.Hour, + } +} + +// messageScope holds the matched messages for a model-filtered analytics +// request, grouped by session. It is a pure value; all DB work happens during +// resolution. +type messageScope struct { + bySession map[string][]ScopedMessage +} + +// resolveAnalyticsMessageScope streams candidate messages for sessionIDs and +// reduces them to the model/time-matched set. It returns nil when no model +// filter is set, signalling the caller to keep its session-grain path. +// includeContent omits the (expensive) content column for count-only panels. +func (db *DB) resolveAnalyticsMessageScope( + ctx context.Context, + sessionIDs []string, + f AnalyticsFilter, + includeContent bool, +) (*messageScope, error) { + if strings.TrimSpace(f.Model) == "" { + return nil, nil + } + + seen := make(map[string]struct{}, len(sessionIDs)) + unique := make([]string, 0, len(sessionIDs)) + for _, id := range sessionIDs { + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + unique = append(unique, id) + } + + flt := f.messageScopeFilter() + loc := f.location() + bySession := make(map[string][]ScopedMessage, len(unique)) + emit := func(m ScopedMessage) { + bySession[m.SessionID] = append(bySession[m.SessionID], m) + } + + contentExpr := "''" + if includeContent { + contentExpr = "COALESCE(content, '')" + } + + if err := queryChunked(unique, func(chunk []string) error { + reducer := NewScopeReducer(flt, emit) + ph, args := inPlaceholders(chunk) + rows, err := db.getReader().QueryContext(ctx, ` + SELECT session_id, ordinal, role, is_system, COALESCE(model, ''), + has_thinking, has_tool_use, COALESCE(timestamp, ''), + output_tokens, has_output_tokens, content_length, `+contentExpr+` + FROM messages + WHERE session_id IN `+ph+` + ORDER BY session_id, ordinal`, + args..., + ) + if err != nil { + return fmt.Errorf("querying analytics candidate messages: %w", err) + } + defer rows.Close() + + for rows.Next() { + var ( + sessionID, role, model, ts, content string + ordinal, outputTokens, contentLength int + isSystem, hasThinking, hasToolUse, hasOutputTokens bool + ) + if err := rows.Scan( + &sessionID, &ordinal, &role, &isSystem, &model, + &hasThinking, &hasToolUse, &ts, &outputTokens, + &hasOutputTokens, &contentLength, &content, + ); err != nil { + return fmt.Errorf("scanning analytics candidate message: %w", err) + } + parsed, has := localTime(ts, loc) + if err := reducer.Push(MessageInput{ + SessionID: sessionID, + Ordinal: ordinal, + Role: role, + Model: model, + IsSystem: isSystem, + Timestamp: ts, + LocalTime: parsed, + HasLocalTime: has, + HasThinking: hasThinking, + HasToolUse: hasToolUse, + OutputTokens: outputTokens, + HasOutputTokens: hasOutputTokens, + ContentLength: contentLength, + Content: content, + }); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating analytics candidate messages: %w", err) + } + return nil + }); err != nil { + return nil, err + } + + return &messageScope{bySession: bySession}, nil +} + +// MessagesBySession returns the matched rows per session. +func (s *messageScope) MessagesBySession() map[string][]ScopedMessage { + return s.bySession +} + +// StatsBySession aggregates matched rows per session. +func (s *messageScope) StatsBySession() map[string]MessageStats { + out := make(map[string]MessageStats, len(s.bySession)) + for id, rows := range s.bySession { + out[id] = ScopeStats(rows) + } + return out +} + +// TimingBySession projects matched rows into the velocity timing view. +func (s *messageScope) TimingBySession() map[string][]TimingMessage { + out := make(map[string][]TimingMessage, len(s.bySession)) + for id, rows := range s.bySession { + out[id] = ScopeTiming(rows) + } + return out +} diff --git a/internal/db/analytics_scope_test.go b/internal/db/analytics_scope_test.go new file mode 100644 index 000000000..73772f2d7 --- /dev/null +++ b/internal/db/analytics_scope_test.go @@ -0,0 +1,195 @@ +package db + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveAnalyticsMessageScope(t *testing.T) { + const ( + sessionA = "scope-sess-a" + sessionB = "scope-sess-b" + model = "claude-sonnet" + otherM = "gpt-4o" + ) + + setup := func(t *testing.T) *DB { + t.Helper() + d := testDB(t) + + insertSession(t, d, sessionA, "proj") + insertSession(t, d, sessionB, "proj") + + // sessionA: user turn then selected-model assistant turn + msgs := []Message{ + { + SessionID: sessionA, + Ordinal: 1, + Role: "user", + Content: "hello content", + ContentLength: len("hello content"), + Timestamp: tsZero, + }, + { + SessionID: sessionA, + Ordinal: 2, + Role: "assistant", + Model: model, + Content: "world content", + ContentLength: len("world content"), + Timestamp: tsZeroS1, + }, + } + + // sessionB: user turn then NON-selected-model assistant turn + msgs = append(msgs, + Message{ + SessionID: sessionB, + Ordinal: 1, + Role: "user", + Content: "ignored user", + ContentLength: len("ignored user"), + Timestamp: tsZero, + }, + Message{ + SessionID: sessionB, + Ordinal: 2, + Role: "assistant", + Model: otherM, + Content: "ignored asst", + ContentLength: len("ignored asst"), + Timestamp: tsZeroS1, + }, + ) + + insertMessages(t, d, msgs...) + return d + } + + t.Run("blank model returns nil", func(t *testing.T) { + d := setup(t) + scope, err := d.resolveAnalyticsMessageScope( + context.Background(), + []string{sessionA}, + AnalyticsFilter{}, + false, + ) + require.NoError(t, err) + assert.Nil(t, scope) + }) + + t.Run("matching session has rows", func(t *testing.T) { + d := setup(t) + scope, err := d.resolveAnalyticsMessageScope( + context.Background(), + []string{sessionA, sessionB}, + AnalyticsFilter{Model: model}, + true, + ) + require.NoError(t, err) + require.NotNil(t, scope) + + bySession := scope.MessagesBySession() + + // sessionA: user + assistant pair both emitted + rowsA := bySession[sessionA] + require.Len(t, rowsA, 2, "sessionA should have 2 matched rows") + assert.Equal(t, "user", rowsA[0].Role) + assert.Equal(t, "assistant", rowsA[1].Role) + + // sessionB: no rows (assistant used a different model) + assert.Empty(t, bySession[sessionB], + "sessionB should yield no rows") + }) + + t.Run("StatsBySession counts user and assistant", func(t *testing.T) { + d := setup(t) + scope, err := d.resolveAnalyticsMessageScope( + context.Background(), + []string{sessionA}, + AnalyticsFilter{Model: model}, + false, + ) + require.NoError(t, err) + require.NotNil(t, scope) + + stats := scope.StatsBySession() + sA, ok := stats[sessionA] + require.True(t, ok, "sessionA stats must exist") + assert.Equal(t, 1, sA.UserMessages, "user message count") + assert.Equal(t, 1, sA.AssistantMessages, "assistant message count") + assert.Equal(t, 2, sA.Messages, "total message count") + }) + + t.Run("TimingBySession returns one entry per row", func(t *testing.T) { + d := setup(t) + scope, err := d.resolveAnalyticsMessageScope( + context.Background(), + []string{sessionA}, + AnalyticsFilter{Model: model}, + false, + ) + require.NoError(t, err) + require.NotNil(t, scope) + + timing := scope.TimingBySession() + tA := timing[sessionA] + assert.Len(t, tA, 2, "timing should have 2 entries") + }) + + t.Run("includeContent=false omits content", func(t *testing.T) { + d := setup(t) + scope, err := d.resolveAnalyticsMessageScope( + context.Background(), + []string{sessionA}, + AnalyticsFilter{Model: model}, + false, + ) + require.NoError(t, err) + require.NotNil(t, scope) + + rows := scope.MessagesBySession()[sessionA] + require.Len(t, rows, 2) + for _, row := range rows { + assert.Empty(t, row.Content, "content should be empty when includeContent=false") + } + }) + + t.Run("includeContent=true populates content", func(t *testing.T) { + d := setup(t) + scope, err := d.resolveAnalyticsMessageScope( + context.Background(), + []string{sessionA}, + AnalyticsFilter{Model: model}, + true, + ) + require.NoError(t, err) + require.NotNil(t, scope) + + rows := scope.MessagesBySession()[sessionA] + require.Len(t, rows, 2) + assert.Equal(t, "hello content", rows[0].Content, + "user row content should be populated") + assert.Equal(t, "world content", rows[1].Content, + "assistant row content should be populated") + }) + + t.Run("deduplicates sessionIDs", func(t *testing.T) { + d := setup(t) + // Pass sessionA twice; resolver must not error or double-count. + scope, err := d.resolveAnalyticsMessageScope( + context.Background(), + []string{sessionA, sessionA}, + AnalyticsFilter{Model: model}, + false, + ) + require.NoError(t, err) + require.NotNil(t, scope) + + rows := scope.MessagesBySession()[sessionA] + assert.Len(t, rows, 2, "dedup should keep exactly 2 rows") + }) +} diff --git a/internal/db/analytics_test.go b/internal/db/analytics_test.go index 702f676f3..6734b557e 100644 --- a/internal/db/analytics_test.go +++ b/internal/db/analytics_test.go @@ -32,16 +32,17 @@ func seedAnalyticsData(t *testing.T, d *DB) seedStats { end string msgs int agent string + model string } sessions := []sessionData{ // Project A: 3 sessions across 2 days, mixed agents - {"a1", "project-alpha", "2024-06-01T09:00:00Z", tsMidYear, 10, "claude"}, - {"a2", "project-alpha", "2024-06-01T14:00:00Z", "2024-06-01T15:00:00Z", 20, "codex"}, - {"a3", "project-alpha", "2024-06-03T09:00:00Z", "2024-06-03T10:00:00Z", 5, "claude"}, + {"a1", "project-alpha", "2024-06-01T09:00:00Z", tsMidYear, 10, "claude", "claude-3-5-sonnet"}, + {"a2", "project-alpha", "2024-06-01T14:00:00Z", "2024-06-01T15:00:00Z", 20, "codex", "gpt-4o"}, + {"a3", "project-alpha", "2024-06-03T09:00:00Z", "2024-06-03T10:00:00Z", 5, "claude", "claude-3-5-sonnet"}, // Project B: 2 sessions on 1 day - {"b1", "project-beta", "2024-06-02T10:00:00Z", "2024-06-02T11:00:00Z", 30, "claude"}, - {"b2", "project-beta", "2024-06-02T15:00:00Z", "2024-06-02T16:00:00Z", 15, "claude"}, + {"b1", "project-beta", "2024-06-02T10:00:00Z", "2024-06-02T11:00:00Z", 30, "claude", "gpt-4o-mini"}, + {"b2", "project-beta", "2024-06-02T15:00:00Z", "2024-06-02T16:00:00Z", 15, "claude", "gpt-4o"}, } stats := seedStats{} @@ -84,6 +85,7 @@ func seedAnalyticsData(t *testing.T, d *DB) seedStats { Content: fmt.Sprintf("msg %d", i), ContentLength: 5, Timestamp: tsMidYear, + Model: sess.model, } } insertMessages(t, d, msgs...) @@ -175,6 +177,15 @@ func TestGetAnalyticsSummary(t *testing.T) { assert.Equal(t, 15, s.MedianMessages, "MedianMessages") // P90 index = int(5*0.9) = 4 → value 30 assert.Equal(t, 30, s.P90Messages, "P90Messages") + assert.Equal(t, + []string{ + "claude-3-5-sonnet", + "gpt-4o", + "gpt-4o-mini", + }, + s.Models, + "Models", + ) require.NotNil(t, s.Agents["claude"], "expected claude agent entry") assert.Equal(t, 4, s.Agents["claude"].Sessions, "claude sessions") @@ -202,6 +213,7 @@ func TestGetAnalyticsSummary(t *testing.T) { t.Run("EmptyDateRange", func(t *testing.T) { s := mustSummary(t, d, ctx, emptyFilter()) assert.Equal(t, 0, s.TotalSessions, "TotalSessions") + assert.Empty(t, s.Models, "Models") }) } @@ -284,6 +296,311 @@ func TestAnalyticsSubagentScope(t *testing.T) { assert.Equal(t, 1, shape.Count, "session-shape stays root-only") } +func TestAnalyticsModelFilter(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "model-a", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:00:00Z") + s.MessageCount = 2 + s.Agent = "claude" + }) + insertMessages(t, d, + Message{ + SessionID: "model-a", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:05:00Z", + Model: "claude-3-5-sonnet", + }, + Message{ + SessionID: "model-a", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T09:06:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + insertSession(t, d, "model-b", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T11:00:00Z") + s.EndedAt = new("2024-06-01T12:00:00Z") + s.MessageCount = 3 + s.Agent = "codex" + }) + insertMessages(t, d, + Message{ + SessionID: "model-b", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T11:05:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "model-b", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T11:06:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "model-b", Ordinal: 2, Role: "user", + Content: "follow-up", ContentLength: 9, + Timestamp: "2024-06-01T11:07:00Z", + Model: "gpt-4o", + }, + ) + + f := AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + } + + t.Run("SingleModel", func(t *testing.T) { + ff := f + ff.Model = "gpt-4o" + s := mustSummary(t, d, ctx, ff) + assert.Equal(t, 1, s.TotalSessions, "TotalSessions") + assert.Equal(t, 3, s.TotalMessages, "TotalMessages") + assert.Equal(t, []string{"gpt-4o"}, s.Models, "Models") + }) + + t.Run("MultiModel", func(t *testing.T) { + ff := f + ff.Model = "gpt-4o, claude-3-5-sonnet" + s := mustSummary(t, d, ctx, ff) + assert.Equal(t, 2, s.TotalSessions, "TotalSessions") + assert.Equal(t, + []string{"claude-3-5-sonnet", "gpt-4o"}, + s.Models, + "Models", + ) + }) +} + +func TestAnalyticsModelFilterGoTimePath(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "dst-claude", "proj", func(s *Session) { + s.StartedAt = new("2026-03-10T14:00:00Z") + s.EndedAt = new("2026-03-10T14:30:00Z") + s.MessageCount = 2 + s.Agent = "claude" + }) + insertMessages(t, d, + Message{ + SessionID: "dst-claude", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2026-03-10T14:05:00Z", + Model: "claude-3-5-sonnet", + }, + Message{ + SessionID: "dst-claude", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2026-03-10T14:06:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + insertSession(t, d, "dst-gpt", "proj", func(s *Session) { + s.StartedAt = new("2026-03-05T15:00:00Z") + s.EndedAt = new("2026-03-05T15:30:00Z") + s.MessageCount = 2 + s.Agent = "codex" + }) + insertMessages(t, d, + Message{ + SessionID: "dst-gpt", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2026-03-05T15:05:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "dst-gpt", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2026-03-05T15:06:00Z", + Model: "gpt-4o", + }, + ) + + hour := 10 + s, err := d.GetAnalyticsSummary(ctx, AnalyticsFilter{ + From: "2026-03-01", + To: "2026-03-31", + Timezone: "America/New_York", + Model: "gpt-4o", + Hour: &hour, + }) + require.NoError(t, err, "GetAnalyticsSummary") + assert.Equal(t, 1, s.TotalSessions, "TotalSessions") + assert.Equal(t, []string{"gpt-4o"}, s.Models, "Models") +} + +func TestAnalyticsSummaryModelFilterCountsOnlyMatchingMessages(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "summary-mixed", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:00:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + }) + insertMessages(t, d, + Message{ + SessionID: "summary-mixed", Ordinal: 0, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T09:05:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "summary-mixed", Ordinal: 1, Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T09:06:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + resp := mustSummary(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + assert.Equal(t, 1, resp.TotalSessions, "TotalSessions") + assert.Equal(t, 1, resp.TotalMessages, "TotalMessages") + assert.Equal(t, []string{"gpt-4o"}, resp.Models, "Models") + assert.Equal(t, 1.0, resp.AvgMessages, "AvgMessages") + assert.Equal(t, 1, resp.MedianMessages, "MedianMessages") + assert.Equal(t, 1, resp.P90Messages, "P90Messages") + require.Contains(t, resp.Agents, "mixed") + assert.Equal(t, 1, resp.Agents["mixed"].Messages, "AgentMessages") +} + +func TestAnalyticsSummaryModelsRespectHourFilterSQLiteSQL(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "hour-a", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T09:30:00Z") + s.MessageCount = 1 + s.Agent = "claude" + }) + insertMessages(t, d, + Message{ + SessionID: "hour-a", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:05:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + insertSession(t, d, "hour-b", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T10:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 1 + s.Agent = "codex" + }) + insertMessages(t, d, + Message{ + SessionID: "hour-b", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T10:05:00Z", + Model: "gpt-4o", + }, + ) + + hour := 9 + s := mustSummary(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Hour: &hour, + }) + assert.Equal(t, 1, s.TotalSessions, "TotalSessions") + assert.Equal(t, + []string{"claude-3-5-sonnet"}, + s.Models, + "Models", + ) +} + +func TestAnalyticsSummaryModelsRespectHourFilterGoPath(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "ktm-a", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T04:15:00Z") + s.EndedAt = new("2024-06-01T04:45:00Z") + s.MessageCount = 1 + s.Agent = "claude" + }) + insertMessages(t, d, + Message{ + SessionID: "ktm-a", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T04:15:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + insertSession(t, d, "ktm-b", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T05:15:00Z") + s.EndedAt = new("2024-06-01T05:45:00Z") + s.MessageCount = 1 + s.Agent = "codex" + }) + insertMessages(t, d, + Message{ + SessionID: "ktm-b", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T05:15:00Z", + Model: "gpt-4o", + }, + ) + + hour := 10 + s := mustSummary(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", + Timezone: "Asia/Kathmandu", + Hour: &hour, + }) + assert.Equal(t, 1, s.TotalSessions, "TotalSessions") + assert.Equal(t, + []string{"claude-3-5-sonnet"}, + s.Models, + "Models", + ) +} + +func TestAnalyticsSummaryModelsUseMatchingHourRowsOnly(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "summary-hour-mixed", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + }) + insertMessages(t, d, + Message{ + SessionID: "summary-hour-mixed", Ordinal: 0, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T09:05:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "summary-hour-mixed", Ordinal: 1, Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T10:05:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + hour := 9 + resp := mustSummary(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Hour: &hour, + }) + assert.Equal(t, 1, resp.TotalSessions, "TotalSessions") + assert.Equal(t, []string{"gpt-4o"}, resp.Models, "Models") +} + func TestAnalyticsFilterMachineMultiSelect(t *testing.T) { d := testDB(t) ctx := context.Background() @@ -367,18 +684,353 @@ func TestGetAnalyticsActivity(t *testing.T) { assert.Equal(t, stats.TotalSessions, resp.Series[0].Sessions, "month sessions") }) - t.Run("HasRoleCounts", func(t *testing.T) { - resp := mustActivity(t, d, ctx, baseFilter(), "day") - totalUser := 0 - totalAsst := 0 - for _, e := range resp.Series { - totalUser += e.UserMessages - totalAsst += e.AssistantMessages - } - assert.Equal(t, stats.TotalMessages, totalUser+totalAsst, "total messages") - assert.Equal(t, stats.TotalUserMessages, totalUser, "total user messages") - assert.Equal(t, stats.TotalAssistantMessages, totalAsst, "total assistant messages") - }) + t.Run("HasRoleCounts", func(t *testing.T) { + resp := mustActivity(t, d, ctx, baseFilter(), "day") + totalUser := 0 + totalAsst := 0 + for _, e := range resp.Series { + totalUser += e.UserMessages + totalAsst += e.AssistantMessages + } + assert.Equal(t, stats.TotalMessages, totalUser+totalAsst, "total messages") + assert.Equal(t, stats.TotalUserMessages, totalUser, "total user messages") + assert.Equal(t, stats.TotalAssistantMessages, totalAsst, "total assistant messages") + }) +} + +func TestGetAnalyticsActivityModelFilter(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "activity-a", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T09:30:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + s.Agent = "claude" + }) + insertMessages(t, d, + Message{ + SessionID: "activity-a", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:00:00Z", + Model: "claude-3-5-sonnet", + }, + Message{ + SessionID: "activity-a", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T09:01:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + insertSession(t, d, "activity-b", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T10:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + s.Agent = "codex" + }) + insertMessages(t, d, + Message{ + SessionID: "activity-b", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T10:00:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "activity-b", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T10:01:00Z", + Model: "gpt-4o", + }, + ) + + resp := mustActivity(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "day") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 2, resp.Series[0].Messages, "Messages") +} + +func TestGetAnalyticsActivityModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "activity-mixed", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T09:30:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + s.Agent = "claude" + }) + insertMessages(t, d, + Message{ + SessionID: "activity-mixed", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:00:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "activity-mixed", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T09:05:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + resp := mustActivity(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "day") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 1, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].UserMessages, "UserMessages") + assert.Equal(t, 0, resp.Series[0].AssistantMessages, + "AssistantMessages") +} + +func TestGetAnalyticsActivityModelFilterKeepsNullTimestampSessionsWithoutTimeFilter( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "activity-null-ts", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T09:30:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + s.Agent = "gpt" + }) + insertMessages(t, d, + Message{ + SessionID: "activity-null-ts", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:00:00Z", + Model: "", + }, + Message{ + SessionID: "activity-null-ts", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Model: "gpt-4o", + }, + ) + + resp := mustActivity(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "day") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 2, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].UserMessages, "UserMessages") + assert.Equal(t, 1, resp.Series[0].AssistantMessages, + "AssistantMessages") +} + +func TestGetAnalyticsActivityModelAndHourFilterUseSameMessage( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "activity-time", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + s.Agent = "claude" + }) + insertMessages(t, d, + Message{ + SessionID: "activity-time", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:00:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "activity-time", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T10:00:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + hour := 10 + resp := mustActivity(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "day") + assert.Empty(t, resp.Series, "Series") +} + +func TestGetAnalyticsActivityModelAndHourFilterKeepsPairedUserTurn( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "activity-paired-hour", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + s.Agent = "claude" + }) + insertMessages(t, d, + Message{ + SessionID: "activity-paired-hour", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:00:00Z", + // Empty model: paired with the gpt-4o assistant at 10:00. Filtering + // by hour 9 must keep the session via this paired user turn, even + // though the model-bearing assistant sits in hour 10. + }, + Message{ + SessionID: "activity-paired-hour", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T10:00:00Z", + Model: "gpt-4o", + }, + ) + + hour := 9 + resp := mustActivity(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "day") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 1, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].UserMessages, "UserMessages") + assert.Equal(t, 0, resp.Series[0].AssistantMessages, + "AssistantMessages") +} + +func TestGetAnalyticsHeatmapSessionsModelAndHourFilterKeepsPairedUserTurn( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "heatmap-sessions-paired", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + }) + insertMessages(t, d, + Message{ + SessionID: "heatmap-sessions-paired", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:00:00Z", + }, + Message{ + SessionID: "heatmap-sessions-paired", Ordinal: 1, + Role: "assistant", Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T10:00:00Z", + Model: "gpt-4o", + }, + ) + + // Empty-model user turn at hour 9 paired with a gpt-4o assistant at hour + // 10. The sessions heatmap must keep the session via the paired user turn. + hour := 9 + resp := mustHeatmap(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "sessions") + require.Len(t, resp.Entries, 1, "len(Entries)") + assert.Equal(t, 1, resp.Entries[0].Value, "Value") +} + +func TestGetAnalyticsTopSessionsDurationModelAndHourFilterKeepsPairedUserTurn( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "top-duration-paired", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:00:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + }) + insertMessages(t, d, + Message{ + SessionID: "top-duration-paired", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:00:00Z", + }, + Message{ + SessionID: "top-duration-paired", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T10:00:00Z", + Model: "gpt-4o", + }, + ) + + // Empty-model user turn at hour 9 paired with a gpt-4o assistant at hour + // 10. Ranking by duration under the gpt-4o + hour-9 filter must keep the + // session via the paired user turn. + hour := 9 + resp, err := d.GetAnalyticsTopSessions(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "duration") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 1, "len(Sessions)") + assert.Equal(t, "top-duration-paired", resp.Sessions[0].ID, "ID") +} + +func TestGetAnalyticsActivityModelAndHourFilterCountsOnlyMatchingHourRows( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "activity-hour-gpt", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.UserMessageCount = 0 + s.Agent = "claude" + }) + + m1 := asstMsgAt("activity-hour-gpt", 0, "[Read: a.go]", "2024-06-01T09:00:00Z") + m1.Model = "gpt-4o" + m1.HasToolUse = true + m1.ToolCalls = []ToolCall{ + {SessionID: "activity-hour-gpt", ToolName: "Read", Category: "Read"}, + {SessionID: "activity-hour-gpt", ToolName: "Bash", Category: "Bash"}, + } + + m2 := asstMsgAt("activity-hour-gpt", 1, "[Grep: b.go]", "2024-06-01T10:00:00Z") + m2.Model = "gpt-4o" + m2.HasToolUse = true + m2.ToolCalls = []ToolCall{ + {SessionID: "activity-hour-gpt", ToolName: "Grep", Category: "Grep"}, + } + + insertMessages(t, d, m1, m2) + + hour := 10 + resp := mustActivity(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "day") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 1, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].AssistantMessages, + "AssistantMessages") + assert.Equal(t, 1, resp.Series[0].ToolCalls, "ToolCalls") } func TestGetAnalyticsHeatmap(t *testing.T) { @@ -453,6 +1105,141 @@ func TestGetAnalyticsHeatmap(t *testing.T) { }) } +func TestGetAnalyticsSummaryModelFilterUsesFilteredOutputTokens( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "summary-output-mixed", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T10:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + s.TotalOutputTokens = 111 + s.HasTotalOutputTokens = true + }) + insertMessages(t, d, + Message{ + SessionID: "summary-output-mixed", Ordinal: 0, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T10:00:00Z", + Model: "gpt-4o", OutputTokens: 11, HasOutputTokens: true, + }, + Message{ + SessionID: "summary-output-mixed", Ordinal: 1, Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T10:05:00Z", + Model: "claude-3-5-sonnet", + OutputTokens: 100, HasOutputTokens: true, + }, + ) + + insertSession(t, d, "summary-output-uncovered", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T10:40:00Z") + s.EndedAt = new("2024-06-01T11:00:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + s.TotalOutputTokens = 90 + s.HasTotalOutputTokens = true + }) + insertMessages(t, d, + Message{ + SessionID: "summary-output-uncovered", Ordinal: 0, + Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T10:40:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "summary-output-uncovered", Ordinal: 1, + Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T10:45:00Z", + Model: "claude-3-5-sonnet", + OutputTokens: 90, HasOutputTokens: true, + }, + ) + + hour := 10 + resp := mustSummary(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }) + assert.Equal(t, 2, resp.TotalSessions, "TotalSessions") + assert.Equal(t, 2, resp.TotalMessages, "TotalMessages") + assert.Equal(t, []string{"gpt-4o"}, resp.Models, "Models") + assert.Equal(t, 11, resp.TotalOutputTokens, "TotalOutputTokens") + assert.Equal(t, 1, resp.TokenReportingSessions, "TokenReportingSessions") +} + +func TestGetAnalyticsHeatmapModelFilterUsesFilteredOutputTokens( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "heatmap-output-mixed", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T10:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + s.TotalOutputTokens = 111 + s.HasTotalOutputTokens = true + }) + insertMessages(t, d, + Message{ + SessionID: "heatmap-output-mixed", Ordinal: 0, + Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T10:00:00Z", + Model: "gpt-4o", OutputTokens: 11, HasOutputTokens: true, + }, + Message{ + SessionID: "heatmap-output-mixed", Ordinal: 1, + Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T10:05:00Z", + Model: "claude-3-5-sonnet", + OutputTokens: 100, HasOutputTokens: true, + }, + ) + + insertSession(t, d, "heatmap-output-uncovered", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T10:40:00Z") + s.EndedAt = new("2024-06-01T11:00:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + s.TotalOutputTokens = 90 + s.HasTotalOutputTokens = true + }) + insertMessages(t, d, + Message{ + SessionID: "heatmap-output-uncovered", Ordinal: 0, + Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T10:40:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "heatmap-output-uncovered", Ordinal: 1, + Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T10:45:00Z", + Model: "claude-3-5-sonnet", + OutputTokens: 90, HasOutputTokens: true, + }, + ) + + hour := 10 + resp := mustHeatmap(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "output_tokens") + require.Len(t, resp.Entries, 1, "len(Entries)") + assert.Equal(t, 11, resp.Entries[0].Value, "Value") +} + func TestGetAnalyticsProjects(t *testing.T) { d := testDB(t) ctx := context.Background() @@ -495,6 +1282,80 @@ func TestGetAnalyticsProjects(t *testing.T) { }) } +func TestGetAnalyticsProjectsModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "projects-mixed", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:00:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + }) + insertMessages(t, d, + Message{ + SessionID: "projects-mixed", Ordinal: 0, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T09:05:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "projects-mixed", Ordinal: 1, Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T09:06:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + resp, err := d.GetAnalyticsProjects(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsProjects") + require.Len(t, resp.Projects, 1, "len(Projects)") + assert.Equal(t, 1, resp.Projects[0].Messages, "Messages") + assert.Equal(t, 1.0, resp.Projects[0].AvgMessages, "AvgMessages") + assert.Equal(t, 1, resp.Projects[0].MedianMessages, "MedianMessages") + assert.Equal(t, 1.0, resp.Projects[0].DailyTrend, "DailyTrend") +} + +func TestGetAnalyticsHeatmapModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "heatmap-mixed", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:00:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + }) + insertMessages(t, d, + Message{ + SessionID: "heatmap-mixed", Ordinal: 0, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T09:05:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "heatmap-mixed", Ordinal: 1, Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T09:06:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + resp := mustHeatmap(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "messages") + require.Len(t, resp.Entries, 1, "len(Entries)") + assert.Equal(t, 1, resp.Entries[0].Value, "Value") +} + func TestMedianInt(t *testing.T) { tests := []struct { name string @@ -841,6 +1702,125 @@ func TestGetAnalyticsHourOfWeek(t *testing.T) { }) } +func TestGetAnalyticsHourOfWeekModelFilter(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "how-a", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T09:30:00Z") + s.MessageCount = 1 + s.UserMessageCount = 1 + }) + insertMessages(t, d, + Message{ + SessionID: "how-a", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:00:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + insertSession(t, d, "how-b", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T10:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 1 + s.UserMessageCount = 1 + }) + insertMessages(t, d, + Message{ + SessionID: "how-b", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T10:00:00Z", + Model: "gpt-4o", + }, + ) + + resp, err := d.GetAnalyticsHourOfWeek(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsHourOfWeek") + assert.Equal(t, 0, findHOWCell(resp.Cells, 5, 9), "Sat 09:00") + assert.Equal(t, 1, findHOWCell(resp.Cells, 5, 10), "Sat 10:00") +} + +func TestGetAnalyticsHourOfWeekModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "how-mixed", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + }) + insertMessages(t, d, + Message{ + SessionID: "how-mixed", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:00:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "how-mixed", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T10:00:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + resp, err := d.GetAnalyticsHourOfWeek(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsHourOfWeek") + assert.Equal(t, 1, findHOWCell(resp.Cells, 5, 9), "Sat 09:00") + assert.Equal(t, 0, findHOWCell(resp.Cells, 5, 10), "Sat 10:00") +} + +func TestGetAnalyticsHourOfWeekModelFilterIncludesPairedUserTurns( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "how-paired", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + }) + insertMessages(t, d, + Message{ + SessionID: "how-paired", Ordinal: 0, Role: "user", + Content: "q", ContentLength: 1, + Timestamp: "2024-06-01T09:00:00Z", + // Empty model: this user turn is paired with the selected-model + // assistant below, so the heatmap must count it like the summary, + // activity, velocity, and trends panels do. + }, + Message{ + SessionID: "how-paired", Ordinal: 1, Role: "assistant", + Content: "a", ContentLength: 1, + Timestamp: "2024-06-01T10:00:00Z", + Model: "gpt-4o", + }, + ) + + resp, err := d.GetAnalyticsHourOfWeek(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsHourOfWeek") + assert.Equal(t, 1, findHOWCell(resp.Cells, 5, 9), + "paired empty-model user turn at Sat 09:00") + assert.Equal(t, 1, findHOWCell(resp.Cells, 5, 10), + "selected-model assistant at Sat 10:00") +} + func findHOWCell(cells []HourOfWeekCell, dow, hour int) int { for _, c := range cells { if c.DayOfWeek == dow && c.Hour == hour { @@ -947,6 +1927,74 @@ func TestGetAnalyticsSessionShape(t *testing.T) { }) } +func TestGetAnalyticsSessionShapeModelFilterUsesMatchingRowsOnly(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "shape-model-filter", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T09:30:00Z") + s.MessageCount = 6 + s.Agent = "mixed" + }) + insertMessages(t, d, + Message{ + SessionID: "shape-model-filter", Ordinal: 0, Role: "user", + Content: "gpt q", ContentLength: 5, + Timestamp: "2024-06-01T09:00:00Z", + }, + Message{ + SessionID: "shape-model-filter", Ordinal: 1, Role: "assistant", + Content: "gpt tool", ContentLength: 8, + Timestamp: "2024-06-01T09:01:00Z", + Model: "gpt-4o", + HasToolUse: true, + ToolCalls: []ToolCall{ + {SessionID: "shape-model-filter", ToolName: "Read", Category: "Read"}, + }, + }, + Message{ + SessionID: "shape-model-filter", Ordinal: 2, Role: "user", + Content: "claude q1", ContentLength: 9, + Timestamp: "2024-06-01T09:02:00Z", + Model: "claude-3-5-sonnet", + }, + Message{ + SessionID: "shape-model-filter", Ordinal: 3, Role: "user", + Content: "claude q2", ContentLength: 9, + Timestamp: "2024-06-01T09:03:00Z", + Model: "claude-3-5-sonnet", + }, + Message{ + SessionID: "shape-model-filter", Ordinal: 4, Role: "user", + Content: "claude q3", ContentLength: 9, + Timestamp: "2024-06-01T09:04:00Z", + Model: "claude-3-5-sonnet", + }, + Message{ + SessionID: "shape-model-filter", Ordinal: 5, Role: "assistant", + Content: "claude reply", ContentLength: 12, + Timestamp: "2024-06-01T09:05:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + resp, err := d.GetAnalyticsSessionShape(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsSessionShape") + assert.Equal(t, 1, resp.Count, "Count") + + lenMap := bucketMap(resp.LengthDistribution) + assert.Equal(t, 1, lenMap["1-5"], "filtered 2-message session stays in 1-5") + assert.Equal(t, 0, lenMap["6-15"], "full-session count must not leak") + + autoMap := bucketMap(resp.AutonomyDistribution) + assert.Equal(t, 1, autoMap["1-2"], "filtered autonomy bucket") + assert.Equal(t, 0, autoMap["<0.5"], "off-model user turns must not leak") +} + func bucketMap( buckets []DistributionBucket, ) map[string]int { @@ -1217,19 +2265,132 @@ func TestGetAnalyticsVelocity_ToolUsage(t *testing.T) { for _, b := range resp.ByAgent { agentMap[b.Label] = b } - assertEq(t, "claude ToolCallsPerActiveMin", agentMap["claude"].Overview.ToolCallsPerActiveMin, 2.0) - assertEq(t, "codex ToolCallsPerActiveMin", agentMap["codex"].Overview.ToolCallsPerActiveMin, 4.0) - }) - - t.Run("ToolCallsPerActiveMinZero", func(t *testing.T) { - d := testDB(t) - insertConversation(t, d, "vt2", "proj", "claude", "2024-06-01T09:00:00Z", []time.Duration{ - 0, 10 * time.Second, + assertEq(t, "claude ToolCallsPerActiveMin", agentMap["claude"].Overview.ToolCallsPerActiveMin, 2.0) + assertEq(t, "codex ToolCallsPerActiveMin", agentMap["codex"].Overview.ToolCallsPerActiveMin, 4.0) + }) + + t.Run("ToolCallsPerActiveMinZero", func(t *testing.T) { + d := testDB(t) + insertConversation(t, d, "vt2", "proj", "claude", "2024-06-01T09:00:00Z", []time.Duration{ + 0, 10 * time.Second, + }) + resp, err := d.GetAnalyticsVelocity(ctx, baseFilter()) + require.NoError(t, err, "GetAnalyticsVelocity") + assertEq(t, "ToolCallsPerActiveMin", resp.Overall.ToolCallsPerActiveMin, 0.0) + }) +} + +func TestGetAnalyticsVelocity_ModelFilterUsesMatchingRowsOnly(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "velocity-model", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T09:11:00Z") + s.MessageCount = 4 + s.Agent = "mixed" + }) + insertMessages(t, d, + Message{ + SessionID: "velocity-model", Ordinal: 0, Role: "user", + Content: "claude q", ContentLength: 8, + Timestamp: "2024-06-01T09:00:00Z", + Model: "claude-3-5-sonnet", + }, + Message{ + SessionID: "velocity-model", Ordinal: 1, Role: "assistant", + Content: "offscope-offscope-xx", ContentLength: 20, + Timestamp: "2024-06-01T09:00:10Z", + Model: "claude-3-5-sonnet", + HasToolUse: true, + ToolCalls: []ToolCall{ + {SessionID: "velocity-model", ToolName: "Read", Category: "Read"}, + {SessionID: "velocity-model", ToolName: "Bash", Category: "Bash"}, + {SessionID: "velocity-model", ToolName: "Grep", Category: "Grep"}, + }, + }, + Message{ + SessionID: "velocity-model", Ordinal: 2, Role: "user", + Content: "gpt q", ContentLength: 5, + Timestamp: "2024-06-01T09:10:00Z", + }, + Message{ + SessionID: "velocity-model", Ordinal: 3, Role: "assistant", + Content: "reply", ContentLength: 5, + Timestamp: "2024-06-01T09:11:00Z", + Model: "gpt-4o", + HasToolUse: true, + ToolCalls: []ToolCall{ + {SessionID: "velocity-model", ToolName: "Edit", Category: "Edit"}, + {SessionID: "velocity-model", ToolName: "Write", Category: "Write"}, + }, + }, + ) + + resp, err := d.GetAnalyticsVelocity(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsVelocity") + assert.Equal(t, 60.0, resp.Overall.FirstResponseSec.P50, + "FirstResponse P50") + assert.Equal(t, 2.0, resp.Overall.MsgsPerActiveMin, + "MsgsPerActiveMin") + assert.Equal(t, 5.0, resp.Overall.CharsPerActiveMin, + "CharsPerActiveMin") + assert.Equal(t, 2.0, resp.Overall.ToolCallsPerActiveMin, + "ToolCallsPerActiveMin") +} + +func TestGetAnalyticsVelocityModelFilterUsesMatchingComplexityBucket(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "velocity-model-complexity", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T09:15:00Z") + s.MessageCount = 16 + s.Agent = "mixed" + }) + msgs := []Message{ + { + SessionID: "velocity-model-complexity", Ordinal: 0, Role: "user", + Content: "gpt q", ContentLength: 5, + Timestamp: "2024-06-01T09:00:00Z", + }, + { + SessionID: "velocity-model-complexity", Ordinal: 1, Role: "assistant", + Content: "reply", ContentLength: 5, + Timestamp: "2024-06-01T09:01:00Z", + Model: "gpt-4o", + }, + } + for i := 2; i < 16; i++ { + role := "user" + if i%2 == 1 { + role = "assistant" + } + msgs = append(msgs, Message{ + SessionID: "velocity-model-complexity", + Ordinal: i, + Role: role, + Content: "claude", + ContentLength: 6, + Timestamp: fmt.Sprintf("2024-06-01T09:%02d:00Z", i), + Model: "claude-3-5-sonnet", }) - resp, err := d.GetAnalyticsVelocity(ctx, baseFilter()) - require.NoError(t, err, "GetAnalyticsVelocity") - assertEq(t, "ToolCallsPerActiveMin", resp.Overall.ToolCallsPerActiveMin, 0.0) + } + insertMessages(t, d, msgs...) + + resp, err := d.GetAnalyticsVelocity(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", }) + require.NoError(t, err, "GetAnalyticsVelocity") + require.Len(t, resp.ByComplexity, 1, "len(ByComplexity)") + assert.Equal(t, "1-15", resp.ByComplexity[0].Label, + "complexity bucket should use filtered message count") + assert.Equal(t, 1, resp.ByComplexity[0].Sessions, "Sessions") } func TestVelocityChunkedQuery(t *testing.T) { @@ -1350,18 +2511,21 @@ func TestGetAnalyticsTools(t *testing.T) { s.MessageCount = 3 s.Agent = "claude" }) - m1 := asstMsg("t1", 0, "[Read: a.go]") + m1 := asstMsgAt("t1", 0, "[Read: a.go]", + "2024-06-01T09:00:00Z") m1.HasToolUse = true m1.ToolCalls = []ToolCall{ {SessionID: "t1", ToolName: "Read", Category: "Read"}, {SessionID: "t1", ToolName: "Read", Category: "Read"}, } - m2 := asstMsg("t1", 1, "[Bash: ls]") + m2 := asstMsgAt("t1", 1, "[Bash: ls]", + "2024-06-01T09:05:00Z") m2.HasToolUse = true m2.ToolCalls = []ToolCall{ {SessionID: "t1", ToolName: "Bash", Category: "Bash"}, } - m3 := asstMsg("t1", 2, "[Edit: b.go]") + m3 := asstMsgAt("t1", 2, "[Edit: b.go]", + "2024-06-01T09:10:00Z") m3.HasToolUse = true m3.ToolCalls = []ToolCall{ {SessionID: "t1", ToolName: "Edit", Category: "Edit"}, @@ -1373,7 +2537,8 @@ func TestGetAnalyticsTools(t *testing.T) { s.MessageCount = 1 s.Agent = "codex" }) - m4 := asstMsg("t2", 0, "[Read: c.go]") + m4 := asstMsgAt("t2", 0, "[Read: c.go]", + "2024-06-02T10:00:00Z") m4.HasToolUse = true m4.ToolCalls = []ToolCall{ {SessionID: "t2", ToolName: "Read", Category: "Read"}, @@ -1454,13 +2619,101 @@ func TestGetAnalyticsTools(t *testing.T) { } func TestAnalyticsToolsToolCallsQueryAggregatesInSQL(t *testing.T) { - q := analyticsToolsQuery("(?,?)") + q := analyticsToolsQuery("(?,?)", "", false) normalized := strings.Join(strings.Fields(strings.ToLower(q)), " ") assert.Contains(t, normalized, - "select session_id, category, count(*)") + "select tc.session_id, tc.category, count(*)") assert.Contains(t, normalized, - "group by session_id, category") + "group by tc.session_id, tc.category") +} + +func TestGetAnalyticsToolsModelFilterCountsOnlyMatchingToolCalls( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "tool-model-1", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.MessageCount = 2 + s.Agent = "claude" + }) + + m1 := asstMsgAt("tool-model-1", 0, "[Read: a.go]", "2024-06-01T09:00:00Z") + m1.Model = "gpt-4o" + m1.HasToolUse = true + m1.ToolCalls = []ToolCall{ + {SessionID: "tool-model-1", ToolName: "Read", Category: "Read"}, + {SessionID: "tool-model-1", ToolName: "Bash", Category: "Bash"}, + } + + m2 := asstMsgAt("tool-model-1", 1, "[Grep: b.go]", "2024-06-01T09:05:00Z") + m2.Model = "claude-3-5-sonnet" + m2.HasToolUse = true + m2.ToolCalls = []ToolCall{ + {SessionID: "tool-model-1", ToolName: "Grep", Category: "Grep"}, + } + + insertMessages(t, d, m1, m2) + + resp, err := d.GetAnalyticsTools(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsTools") + assert.Equal(t, 2, resp.TotalCalls, "TotalCalls") + require.Len(t, resp.ByCategory, 2, "len(ByCategory)") + + catMap := make(map[string]int) + for _, c := range resp.ByCategory { + catMap[c.Category] = c.Count + } + assert.Equal(t, 1, catMap["Read"], "Read") + assert.Equal(t, 1, catMap["Bash"], "Bash") + assert.Zero(t, catMap["Grep"], "Grep") +} + +func TestGetAnalyticsToolsModelAndHourFilterCountsOnlyMatchingHourToolCalls( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "tool-model-hour", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:30:00Z") + s.MessageCount = 2 + s.Agent = "claude" + }) + + m1 := asstMsgAt("tool-model-hour", 0, "[Read: a.go]", "2024-06-01T09:00:00Z") + m1.Model = "gpt-4o" + m1.HasToolUse = true + m1.ToolCalls = []ToolCall{ + {SessionID: "tool-model-hour", ToolName: "Read", Category: "Read"}, + {SessionID: "tool-model-hour", ToolName: "Bash", Category: "Bash"}, + } + + m2 := asstMsgAt("tool-model-hour", 1, "[Grep: b.go]", "2024-06-01T10:00:00Z") + m2.Model = "gpt-4o" + m2.HasToolUse = true + m2.ToolCalls = []ToolCall{ + {SessionID: "tool-model-hour", ToolName: "Grep", Category: "Grep"}, + } + + insertMessages(t, d, m1, m2) + + hour := 10 + resp, err := d.GetAnalyticsTools(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }) + require.NoError(t, err, "GetAnalyticsTools") + assert.Equal(t, 1, resp.TotalCalls, "TotalCalls") + require.Len(t, resp.ByCategory, 1, "len(ByCategory)") + assert.Equal(t, "Grep", resp.ByCategory[0].Category, "Category") + assert.Equal(t, 1, resp.ByCategory[0].Count, "Count") } func TestGetAnalyticsSkills(t *testing.T) { @@ -1834,7 +3087,7 @@ func TestGetAnalyticsSkillsDateBoundaries(t *testing.T) { } func TestAnalyticsSkillsToolCallsQueryAggregatesInSQL(t *testing.T) { - q := analyticsSkillsQuery("(?,?)") + q := analyticsSkillsQuery("(?,?)", "") normalized := strings.Join(strings.Fields(strings.ToLower(q)), " ") assert.Contains(t, normalized, @@ -1850,6 +3103,58 @@ func TestAnalyticsSkillsToolCallsQueryAggregatesInSQL(t *testing.T) { "coalesce(m.timestamp, '')") } +func TestGetAnalyticsSkillsModelFilterCountsOnlyMatchingSkillCalls( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "skill-model-1", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.MessageCount = 2 + s.Agent = "claude" + }) + + m1 := asstMsgAt("skill-model-1", 0, "skill call", "2024-06-01T09:00:00Z") + m1.Model = "gpt-4o" + m1.HasToolUse = true + m1.ToolCalls = []ToolCall{ + { + SessionID: "skill-model-1", + ToolName: "Skill", + Category: "Skill", + SkillName: "review-code", + }, + } + + m2 := asstMsgAt("skill-model-1", 1, "skill call", "2024-06-01T09:05:00Z") + m2.Model = "claude-3-5-sonnet" + m2.HasToolUse = true + m2.ToolCalls = []ToolCall{ + { + SessionID: "skill-model-1", + ToolName: "Skill", + Category: "Skill", + SkillName: "write-tests", + }, + } + + insertMessages(t, d, m1, m2) + + resp, err := d.GetAnalyticsSkills(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsSkills") + assert.Equal(t, 1, resp.TotalSkillCalls, "TotalSkillCalls") + assert.Equal(t, 1, resp.DistinctSkills, "DistinctSkills") + require.Len(t, resp.BySkill, 1, "len(BySkill)") + assert.Equal(t, "review-code", resp.BySkill[0].SkillName, "SkillName") + assert.Equal(t, 1, resp.BySkill[0].CallCount, "CallCount") + assert.Equal(t, "2024-06-01T09:00:00Z", resp.BySkill[0].LastUsedAt, + "LastUsedAt") +} + func TestGetAnalyticsToolsCanceled(t *testing.T) { d := testDB(t) ctx := canceledCtx() @@ -1894,6 +3199,42 @@ func TestActivityToolAndThinkingCounts(t *testing.T) { assert.Equal(t, 2, entry.ToolCalls, "ToolCalls") } +func TestActivityToolCallsRespectModelFilter(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "at-model", "proj", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.MessageCount = 2 + s.Agent = "claude" + }) + + m1 := asstMsgAt("at-model", 0, "[Read: a.go]", "2024-06-01T09:00:00Z") + m1.Model = "gpt-4o" + m1.HasToolUse = true + m1.ToolCalls = []ToolCall{ + {SessionID: "at-model", ToolName: "Read", Category: "Read"}, + {SessionID: "at-model", ToolName: "Bash", Category: "Bash"}, + } + + m2 := asstMsgAt("at-model", 1, "[Grep: b.go]", "2024-06-01T09:05:00Z") + m2.Model = "claude-3-5-sonnet" + m2.HasToolUse = true + m2.ToolCalls = []ToolCall{ + {SessionID: "at-model", ToolName: "Grep", Category: "Grep"}, + } + + insertMessages(t, d, m1, m2) + + resp := mustActivity(t, d, ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "day") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Messages, "Messages") + assert.Equal(t, 2, resp.Series[0].ToolCalls, "ToolCalls") +} + func TestGetAnalyticsTopSessions(t *testing.T) { d := testDB(t) ctx := context.Background() @@ -2351,6 +3692,92 @@ func TestGetAnalyticsTopSessions(t *testing.T) { }) } +func TestGetAnalyticsTopSessionsOutputTokensUseFilteredModelTotals( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "top-output-mixed", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:00:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + s.TotalOutputTokens = 510 + s.HasTotalOutputTokens = true + }) + insertMessages(t, d, + Message{ + SessionID: "top-output-mixed", Ordinal: 0, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T09:00:00Z", + Model: "gpt-4o", OutputTokens: 10, HasOutputTokens: true, + }, + Message{ + SessionID: "top-output-mixed", Ordinal: 1, Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T09:05:00Z", + Model: "claude-3-5-sonnet", + OutputTokens: 500, HasOutputTokens: true, + }, + ) + + insertSession(t, d, "top-output-gpt", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T11:00:00Z") + s.EndedAt = new("2024-06-01T12:00:00Z") + s.MessageCount = 1 + s.Agent = "gpt" + s.TotalOutputTokens = 30 + s.HasTotalOutputTokens = true + }) + insertMessages(t, d, Message{ + SessionID: "top-output-gpt", Ordinal: 0, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T11:00:00Z", + Model: "gpt-4o", OutputTokens: 30, HasOutputTokens: true, + }) + + insertSession(t, d, "top-output-uncovered", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T13:00:00Z") + s.EndedAt = new("2024-06-01T14:00:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + s.TotalOutputTokens = 900 + s.HasTotalOutputTokens = true + }) + insertMessages(t, d, + Message{ + SessionID: "top-output-uncovered", Ordinal: 0, + Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T13:00:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "top-output-uncovered", Ordinal: 1, + Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T13:05:00Z", + Model: "claude-3-5-sonnet", + OutputTokens: 900, HasOutputTokens: true, + }, + ) + + resp, err := d.GetAnalyticsTopSessions(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "output_tokens") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 2, "len(Sessions)") + assert.Equal(t, "top-output-gpt", resp.Sessions[0].ID, "top session") + assert.Equal(t, 30, resp.Sessions[0].OutputTokens, + "top OutputTokens") + assert.Equal(t, "top-output-mixed", resp.Sessions[1].ID, + "second session") + assert.Equal(t, 10, resp.Sessions[1].OutputTokens, + "second OutputTokens") +} + func TestGetAnalyticsTopSessionsDisplayName(t *testing.T) { d := testDB(t) ctx := context.Background() @@ -2402,6 +3829,121 @@ func TestGetAnalyticsTopSessionsDisplayName(t *testing.T) { assert.Equal(t, customName, *custom.DisplayName) } +func TestGetAnalyticsTopSessionsMessagesUseFilteredModelCounts( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "top-mixed", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T10:00:00Z") + s.MessageCount = 3 + s.Agent = "mixed" + }) + insertMessages(t, d, + Message{ + SessionID: "top-mixed", Ordinal: 0, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T09:00:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "top-mixed", Ordinal: 1, Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T09:05:00Z", + Model: "claude-3-5-sonnet", + }, + Message{ + SessionID: "top-mixed", Ordinal: 2, Role: "assistant", + Content: "claude", ContentLength: 6, + Timestamp: "2024-06-01T09:06:00Z", + Model: "claude-3-5-sonnet", + }, + ) + + insertSession(t, d, "top-gpt", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T11:00:00Z") + s.EndedAt = new("2024-06-01T12:00:00Z") + s.MessageCount = 2 + s.Agent = "gpt" + }) + insertMessages(t, d, + Message{ + SessionID: "top-gpt", Ordinal: 0, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T11:00:00Z", + Model: "gpt-4o", + }, + Message{ + SessionID: "top-gpt", Ordinal: 1, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T11:05:00Z", + Model: "gpt-4o", + }, + ) + + resp, err := d.GetAnalyticsTopSessions(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "messages") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 2, "len(Sessions)") + assert.Equal(t, "top-gpt", resp.Sessions[0].ID, "top session") + assert.Equal(t, 2, resp.Sessions[0].MessageCount, "top MessageCount") + assert.Equal(t, "top-mixed", resp.Sessions[1].ID, "second session") + assert.Equal(t, 1, resp.Sessions[1].MessageCount, "second MessageCount") +} + +func TestGetAnalyticsTopSessionsModelFilterCapsAtTen(t *testing.T) { + d := testDB(t) + ctx := context.Background() + + // Twelve sessions all match the gpt-4o filter, so the model-scoped + // re-sort drops the SQL LIMIT and ranks every matching session. The + // caller must still cap the result at the top ten. Session i carries + // i+1 gpt-4o assistant messages of ten output tokens each, so + // "top-cap-11" has the most messages and the most tokens. + for i := range 12 { + id := fmt.Sprintf("top-cap-%d", i) + count := i + 1 + hour := fmt.Sprintf("%02d", 8+i) + insertSession(t, d, id, "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T" + hour + ":00:00Z") + s.EndedAt = new("2024-06-01T" + hour + ":30:00Z") + s.MessageCount = count + s.Agent = "gpt" + s.TotalOutputTokens = count * 10 + s.HasTotalOutputTokens = true + }) + msgs := make([]Message, count) + for j := range msgs { + msgs[j] = Message{ + SessionID: id, Ordinal: j, Role: "assistant", + Content: "gpt", ContentLength: 3, + Timestamp: "2024-06-01T" + hour + ":00:00Z", + Model: "gpt-4o", + OutputTokens: 10, HasOutputTokens: true, + } + } + insertMessages(t, d, msgs...) + } + + for _, metric := range []string{"messages", "output_tokens"} { + t.Run(metric, func(t *testing.T) { + resp, err := d.GetAnalyticsTopSessions(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, metric) + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 10, + "model-filtered top sessions capped at ten") + assert.Equal(t, "top-cap-11", resp.Sessions[0].ID, + "highest-count session ranks first") + }) + } +} + func TestBuildWhereProjectFilter(t *testing.T) { d := testDB(t) ctx := context.Background() @@ -3187,6 +4729,110 @@ func TestGetAnalyticsSignalSessionsRejectsUnsupportedSignal(t *testing.T) { } } +func TestGetAnalyticsSignalSessionsModelFilterUsesMatchingMessages( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "signal-mixed", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T09:10:00Z") + s.MessageCount = 2 + s.Agent = "mixed" + }) + insertMessages(t, d, + Message{ + SessionID: "signal-mixed", Ordinal: 0, Role: "assistant", + Content: "claude tool evidence", ContentLength: 20, + Timestamp: "2024-06-01T09:05:00Z", + Model: "claude-3-5-sonnet", + HasToolUse: true, + }, + Message{ + SessionID: "signal-mixed", Ordinal: 1, Role: "assistant", + Content: "gpt tool evidence", ContentLength: 17, + Timestamp: "2024-06-01T09:06:00Z", + Model: "gpt-4o", + HasToolUse: true, + }, + ) + require.NoError(t, d.UpdateSessionSignals( + "signal-mixed", + SessionSignalUpdate{ToolFailureSignalCount: 1}, + )) + + resp, err := d.GetAnalyticsSignalSessions( + ctx, + AnalyticsFilter{ + From: "2024-06-01", + To: "2024-06-01", + Timezone: "UTC", + Model: "gpt-4o", + }, + "tool_failure_signals", + 10, + ) + require.NoError(t, err, "GetAnalyticsSignalSessions") + require.Len(t, resp.Sessions, 1, "len(Sessions)") + assert.Equal(t, "gpt tool evidence", resp.Sessions[0].Excerpt) + require.NotNil(t, resp.Sessions[0].MessageOrdinal) + assert.Equal(t, 1, *resp.Sessions[0].MessageOrdinal) +} + +func TestGetAnalyticsSignalSessionsModelFilterKeepsParserUserEvidence( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + + insertSession(t, d, "signal-parser-user", "alpha", func(s *Session) { + s.StartedAt = new("2024-06-01T09:00:00Z") + s.EndedAt = new("2024-06-01T09:10:00Z") + s.MessageCount = 2 + s.UserMessageCount = 1 + s.Agent = "mixed" + }) + insertMessages(t, d, + Message{ + SessionID: "signal-parser-user", Ordinal: 0, Role: "user", + Content: "help", ContentLength: 4, + Timestamp: "2024-06-01T09:00:00Z", + Model: "", + }, + Message{ + SessionID: "signal-parser-user", Ordinal: 1, Role: "assistant", + Content: "reply", ContentLength: 5, + Timestamp: "2024-06-01T09:01:00Z", + Model: "gpt-4o", + HasToolUse: true, + }, + ) + updateSignals(t, d, "signal-parser-user", SessionSignalUpdate{ + QualitySignals: QualitySignals{ + Version: 1, + ShortPromptCount: 1, + }, + }) + + resp, err := d.GetAnalyticsSignalSessions( + ctx, + AnalyticsFilter{ + From: "2024-06-01", + To: "2024-06-01", + Timezone: "UTC", + Model: "gpt-4o", + }, + "short_prompt_count", + 10, + ) + require.NoError(t, err, "GetAnalyticsSignalSessions") + require.Len(t, resp.Sessions, 1, "len(Sessions)") + assert.Equal(t, "help", resp.Sessions[0].Excerpt) + require.NotNil(t, resp.Sessions[0].MessageOrdinal) + assert.Equal(t, 0, *resp.Sessions[0].MessageOrdinal) +} + func TestParseEvidenceTimeAcceptsPostgresUTCFormat(t *testing.T) { got, ok := parseEvidenceTime("2024-06-01T12:34:56.123456Z") if !ok { diff --git a/internal/db/messagescope.go b/internal/db/messagescope.go new file mode 100644 index 000000000..cd42cafcf --- /dev/null +++ b/internal/db/messagescope.go @@ -0,0 +1,155 @@ +package db + +import ( + "sort" + "time" +) + +// Model-scoped analytics core. The matched-message reducer +// (messagescope_reducer.go) and these projections turn a backend's candidate +// message rows into per-session stats and timing. The user-turn pairing +// semantics live here so analytics and trends across the SQLite, PostgreSQL, +// and DuckDB backends share one implementation instead of duplicating it per +// panel. This code is pure (no SQL, timezone, or AnalyticsFilter dependency) +// and is unit-tested without a database in messagescope_test.go. + +// ScopeFilter is the message-grain predicate, adapted by each backend from +// AnalyticsFilter. Models holds the already-split model CSV. DayOfWeek and Hour +// are the day/hour match targets (nil = unconstrained); DayOfWeek uses ISO +// numbering Monday=0..Sunday=6. +type ScopeFilter struct { + Models map[string]struct{} + DayOfWeek *int + Hour *int +} + +// MessageInput is one candidate row, already filtered by the backend's +// panel-specific predicate and already time-localized. Rows MUST be pushed +// grouped by SessionID (each session contiguous) with non-decreasing Ordinal +// within each session; cross-session order is unconstrained. See +// ScopeReducer.Push. +type MessageInput struct { + SessionID string + Ordinal int + Role string + Model string + IsSystem bool + Timestamp string + LocalTime time.Time + HasLocalTime bool + HasThinking bool + HasToolUse bool + OutputTokens int + HasOutputTokens bool + ContentLength int + Content string +} + +// ScopedMessage is a matched, emitted row. It preserves the raw Timestamp +// string for downstream timing (e.g. Signals evidence) alongside the localized +// time used for the day/hour match. +type ScopedMessage struct { + SessionID string + Ordinal int + Role string + Content string + IsSystem bool + HasThinking bool + HasToolUse bool + Timestamp string + LocalTime time.Time + HasLocalTime bool + OutputTokens int + HasOutputTokens bool + ContentLength int +} + +// MessageStats is the per-session aggregate of matched rows. +type MessageStats struct { + Messages int + UserMessages int + AssistantMessages int + ToolUseMessages int + ThinkingMessages int + OutputTokens int + HasOutputTokens bool +} + +// TimingMessage is the velocity projection of a matched row. +type TimingMessage struct { + Role string + Time time.Time + Valid bool + ContentLength int +} + +// MatchesDayHour reports whether a localized time satisfies the day/hour +// filter. With no constraint it always matches (even when the time is +// unparsed); with a constraint, an unparsed time never matches. +func (f ScopeFilter) MatchesDayHour(t time.Time, has bool) bool { + if f.DayOfWeek == nil && f.Hour == nil { + return true + } + if !has { + return false + } + if f.DayOfWeek != nil && (int(t.Weekday())+6)%7 != *f.DayOfWeek { + return false + } + if f.Hour != nil && t.Hour() != *f.Hour { + return false + } + return true +} + +// ScopeStats aggregates matched rows into MessageStats. The caller groups rows +// by session before calling. +func ScopeStats(rows []ScopedMessage) MessageStats { + var s MessageStats + for _, row := range rows { + s.Messages++ + switch row.Role { + case "user": + if !row.IsSystem { + s.UserMessages++ + } + case "assistant": + s.AssistantMessages++ + if row.HasToolUse { + s.ToolUseMessages++ + } + } + if row.HasThinking { + s.ThinkingMessages++ + } + if row.HasOutputTokens { + s.OutputTokens += row.OutputTokens + s.HasOutputTokens = true + } + } + return s +} + +// ScopeTiming projects matched rows into the velocity timing view in ordinal +// (conversation) order. The reducer emits a selected non-assistant row ahead of +// an earlier buffered user turn (see ScopeReducer), so emitted rows can be out +// of ordinal order. Velocity pairs each prompt with the following response by +// position, so the timing view is re-sorted by ordinal here without disturbing +// the emit order the message and stats projections rely on. +func ScopeTiming(rows []ScopedMessage) []TimingMessage { + ordered := make([]ScopedMessage, len(rows)) + copy(ordered, rows) + sort.SliceStable(ordered, func(i, j int) bool { + return ordered[i].Ordinal < ordered[j].Ordinal + }) + out := make([]TimingMessage, 0, len(ordered)) + for _, row := range ordered { + out = append(out, TimingMessage{ + Role: row.Role, + Time: row.LocalTime, + Valid: row.HasLocalTime, + ContentLength: row.ContentLength, + }) + } + return out +} diff --git a/internal/db/messagescope_reducer.go b/internal/db/messagescope_reducer.go new file mode 100644 index 000000000..08668d2c5 --- /dev/null +++ b/internal/db/messagescope_reducer.go @@ -0,0 +1,115 @@ +package db + +import "fmt" + +// ScopeReducer applies model membership, user-turn pairing, and the day/hour +// match to a stream of candidate rows, calling emit for each matched +// ScopedMessage. Emit order mirrors the reference +// getAnalyticsModelScopedMessages (analytics.go): buffered user turns are +// flushed only when their selected assistant arrives, so a selected +// non-assistant row that lands between a pending user and that assistant is +// emitted ahead of the user. Changing this ordering is a cross-backend behavior +// change, not a local fix. +// +// Candidate rows MUST be grouped by session (every row of a session +// contiguous) with non-decreasing Ordinal within each session -- exactly what +// SQL "ORDER BY session_id, ordinal" yields under any collation. Cross-session +// order is unconstrained, so the reducer never assumes Go byte order matches +// the database collation. A session reappearing after its group ended, or an +// ordinal moving backwards within a session, is a query bug and is returned as +// an error (never a panic), since this runs in request handling. +type ScopeReducer struct { + filter ScopeFilter + emit func(ScopedMessage) + session string + lastOrd int + started bool + pending []ScopedMessage + seen map[string]struct{} +} + +// NewScopeReducer returns a ScopeReducer that calls emit for each matched row. +func NewScopeReducer(f ScopeFilter, emit func(ScopedMessage)) *ScopeReducer { + return &ScopeReducer{filter: f, emit: emit, seen: make(map[string]struct{})} +} + +// Push feeds one candidate row. O(1) grouping check, no allocation beyond the +// pending buffer (and one map entry per completed session). +func (r *ScopeReducer) Push(row MessageInput) error { + switch { + case !r.started: + r.started = true + r.session = row.SessionID + case row.SessionID == r.session: + if row.Ordinal < r.lastOrd { + return fmt.Errorf( + "db: scope ordinal out of order in session %q: %d after %d", + row.SessionID, row.Ordinal, r.lastOrd, + ) + } + default: + if _, done := r.seen[row.SessionID]; done { + return fmt.Errorf( + "db: scope session %q reappeared; candidate rows must be grouped by session", + row.SessionID, + ) + } + r.seen[r.session] = struct{}{} + r.session = row.SessionID + r.pending = r.pending[:0] + } + r.lastOrd = row.Ordinal + + scoped := scopedFrom(row) + if row.Role == "user" && !row.IsSystem && row.Model == "" { + r.pending = append(r.pending, scoped) + return nil + } + _, selected := r.filter.Models[row.Model] + switch row.Role { + case "assistant": + if selected { + r.flush() + r.appendMatched(scoped) + return nil + } + r.pending = r.pending[:0] + default: + if selected { + r.appendMatched(scoped) + } + } + return nil +} + +func (r *ScopeReducer) flush() { + for _, row := range r.pending { + r.appendMatched(row) + } + r.pending = r.pending[:0] +} + +func (r *ScopeReducer) appendMatched(row ScopedMessage) { + if !r.filter.MatchesDayHour(row.LocalTime, row.HasLocalTime) { + return + } + r.emit(row) +} + +func scopedFrom(row MessageInput) ScopedMessage { + return ScopedMessage{ + SessionID: row.SessionID, + Ordinal: row.Ordinal, + Role: row.Role, + Content: row.Content, + IsSystem: row.IsSystem, + HasThinking: row.HasThinking, + HasToolUse: row.HasToolUse, + Timestamp: row.Timestamp, + LocalTime: row.LocalTime, + HasLocalTime: row.HasLocalTime, + OutputTokens: row.OutputTokens, + HasOutputTokens: row.HasOutputTokens, + ContentLength: row.ContentLength, + } +} diff --git a/internal/db/messagescope_reducer_test.go b/internal/db/messagescope_reducer_test.go new file mode 100644 index 000000000..c16a9f750 --- /dev/null +++ b/internal/db/messagescope_reducer_test.go @@ -0,0 +1,152 @@ +package db + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// collectScopeRows runs rows through a reducer and returns emitted rows. +func collectScopeRows(t *testing.T, f ScopeFilter, rows []MessageInput) ([]ScopedMessage, error) { + t.Helper() + var out []ScopedMessage + r := NewScopeReducer(f, func(m ScopedMessage) { out = append(out, m) }) + for _, row := range rows { + if err := r.Push(row); err != nil { + return out, err + } + } + return out, nil +} + +func scopeUser(session string, ord int) MessageInput { + return MessageInput{SessionID: session, Ordinal: ord, Role: "user", HasLocalTime: true} +} + +func scopeAssistant(session string, ord int, model string) MessageInput { + return MessageInput{SessionID: session, Ordinal: ord, Role: "assistant", Model: model, HasLocalTime: true} +} + +func TestScopeReducerPairing(t *testing.T) { + f := ScopeFilter{Models: map[string]struct{}{"sonnet": {}}} + + t.Run("selected assistant flushes preceding user", func(t *testing.T) { + out, err := collectScopeRows(t, f, []MessageInput{scopeUser("s1", 0), scopeAssistant("s1", 1, "sonnet")}) + require.NoError(t, err) + require.Len(t, out, 2) + assert.Equal(t, "user", out[0].Role) + assert.Equal(t, "assistant", out[1].Role) + }) + + t.Run("non-selected assistant drops preceding user", func(t *testing.T) { + out, err := collectScopeRows(t, f, []MessageInput{scopeUser("s1", 0), scopeAssistant("s1", 1, "opus")}) + require.NoError(t, err) + assert.Empty(t, out) + }) + + t.Run("session change drops pending", func(t *testing.T) { + out, err := collectScopeRows(t, f, []MessageInput{scopeUser("s1", 0), scopeAssistant("s2", 0, "sonnet")}) + require.NoError(t, err) + require.Len(t, out, 1) + assert.Equal(t, "s2", out[0].SessionID) + }) + + t.Run("day/hour filter drops both user and assistant", func(t *testing.T) { + ff := ScopeFilter{Models: map[string]struct{}{"sonnet": {}}, Hour: new(9)} + mon14 := time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC) + rows := []MessageInput{ + {SessionID: "s1", Ordinal: 0, Role: "user", HasLocalTime: true, LocalTime: mon14}, + {SessionID: "s1", Ordinal: 1, Role: "assistant", Model: "sonnet", HasLocalTime: true, LocalTime: mon14}, + } + out, err := collectScopeRows(t, ff, rows) + require.NoError(t, err) + assert.Empty(t, out) + }) + + // The day/hour match applies per message, not per pair: a flushed user and + // its triggering assistant are filtered independently. + t.Run("day/hour filter keeps matching user, drops non-matching assistant", func(t *testing.T) { + ff := ScopeFilter{Models: map[string]struct{}{"sonnet": {}}, Hour: new(9)} + mon09 := time.Date(2024, 1, 1, 9, 0, 0, 0, time.UTC) + mon14 := time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC) + rows := []MessageInput{ + {SessionID: "s1", Ordinal: 0, Role: "user", HasLocalTime: true, LocalTime: mon09}, + {SessionID: "s1", Ordinal: 1, Role: "assistant", Model: "sonnet", HasLocalTime: true, LocalTime: mon14}, + } + out, err := collectScopeRows(t, ff, rows) + require.NoError(t, err) + require.Len(t, out, 1) + assert.Equal(t, "user", out[0].Role) + }) + + t.Run("day/hour filter drops non-matching user, keeps matching assistant", func(t *testing.T) { + ff := ScopeFilter{Models: map[string]struct{}{"sonnet": {}}, Hour: new(9)} + mon09 := time.Date(2024, 1, 1, 9, 0, 0, 0, time.UTC) + mon14 := time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC) + rows := []MessageInput{ + {SessionID: "s1", Ordinal: 0, Role: "user", HasLocalTime: true, LocalTime: mon14}, + {SessionID: "s1", Ordinal: 1, Role: "assistant", Model: "sonnet", HasLocalTime: true, LocalTime: mon09}, + } + out, err := collectScopeRows(t, ff, rows) + require.NoError(t, err) + require.Len(t, out, 1) + assert.Equal(t, "assistant", out[0].Role) + }) + + // A selected non-assistant row interleaved with a still-pending user emits + // immediately, ahead of the buffered user (ordinals 1, 0, 2). This pins + // parity with the reference getAnalyticsModelScopedMessages default case + // (analytics.go); changing it is a cross-backend behavior change that must + // update all backends and the oracle together. + t.Run("selected non-assistant row emits ahead of pending user (reference parity)", func(t *testing.T) { + rows := []MessageInput{ + {SessionID: "s1", Ordinal: 0, Role: "user", HasLocalTime: true}, + {SessionID: "s1", Ordinal: 1, Role: "user", IsSystem: true, Model: "sonnet", HasLocalTime: true}, + {SessionID: "s1", Ordinal: 2, Role: "assistant", Model: "sonnet", HasLocalTime: true}, + } + out, err := collectScopeRows(t, f, rows) + require.NoError(t, err) + require.Len(t, out, 3) + assert.Equal(t, []int{1, 0, 2}, []int{out[0].Ordinal, out[1].Ordinal, out[2].Ordinal}) + }) +} + +func TestScopeReducerOrdering(t *testing.T) { + f := ScopeFilter{Models: map[string]struct{}{"sonnet": {}}} + + t.Run("decreasing ordinal within session errors", func(t *testing.T) { + _, err := collectScopeRows(t, f, []MessageInput{scopeAssistant("s1", 5, "sonnet"), scopeAssistant("s1", 2, "sonnet")}) + require.Error(t, err) + }) + + // Sessions need only be grouped, not byte-ordered. PostgreSQL "ORDER BY + // session_id" under a non-C collation can hand the reducer groups in an + // order Go string comparison calls "backwards"; that is valid input and + // must not error, or model-filtered analytics would fail on those rows. + t.Run("non-byte-ordered session groups are accepted", func(t *testing.T) { + out, err := collectScopeRows(t, f, []MessageInput{ + scopeUser("s2", 0), scopeAssistant("s2", 1, "sonnet"), + scopeUser("s1", 0), scopeAssistant("s1", 1, "sonnet"), + }) + require.NoError(t, err) + require.Len(t, out, 4) + assert.Equal(t, "s2", out[0].SessionID) + assert.Equal(t, "s1", out[2].SessionID) + }) + + t.Run("session reappearing after its group ends errors", func(t *testing.T) { + _, err := collectScopeRows(t, f, []MessageInput{ + scopeAssistant("s1", 0, "sonnet"), + scopeAssistant("s2", 0, "sonnet"), + scopeAssistant("s1", 1, "sonnet"), + }) + require.Error(t, err) + }) + + t.Run("grouped ascending stream is fine", func(t *testing.T) { + _, err := collectScopeRows(t, f, []MessageInput{scopeAssistant("s1", 0, "sonnet"), scopeAssistant("s1", 1, "sonnet"), scopeAssistant("s2", 0, "sonnet")}) + require.NoError(t, err) + }) +} diff --git a/internal/db/messagescope_test.go b/internal/db/messagescope_test.go new file mode 100644 index 000000000..a57de2b6d --- /dev/null +++ b/internal/db/messagescope_test.go @@ -0,0 +1,92 @@ +package db + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScopeFilterMatchesDayHour(t *testing.T) { + // 2024-01-01 is a Monday (ISO day 0); hour 14 UTC. + mon14 := time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC) + tests := []struct { + name string + f ScopeFilter + t time.Time + has bool + want bool + }{ + {"no constraint matches even unparsed", ScopeFilter{}, time.Time{}, false, true}, + {"day match", ScopeFilter{DayOfWeek: new(0)}, mon14, true, true}, + {"day mismatch", ScopeFilter{DayOfWeek: new(1)}, mon14, true, false}, + {"hour match", ScopeFilter{Hour: new(14)}, mon14, true, true}, + {"hour mismatch", ScopeFilter{Hour: new(9)}, mon14, true, false}, + {"constraint but unparsed never matches", ScopeFilter{Hour: new(14)}, time.Time{}, false, false}, + {"day and hour both match", ScopeFilter{DayOfWeek: new(0), Hour: new(14)}, mon14, true, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, tc.f.MatchesDayHour(tc.t, tc.has)) + }) + } +} + +func TestScopeStats(t *testing.T) { + rows := []ScopedMessage{ + {Role: "user"}, + {Role: "user", IsSystem: true}, + {Role: "assistant", HasToolUse: true, HasThinking: true, HasOutputTokens: true, OutputTokens: 10}, + {Role: "assistant", HasOutputTokens: true, OutputTokens: 5}, + } + s := ScopeStats(rows) + assert.Equal(t, 4, s.Messages) + assert.Equal(t, 1, s.UserMessages) // system user not counted + assert.Equal(t, 2, s.AssistantMessages) + assert.Equal(t, 1, s.ToolUseMessages) + assert.Equal(t, 1, s.ThinkingMessages) + assert.Equal(t, 15, s.OutputTokens) + assert.True(t, s.HasOutputTokens) +} + +func TestScopeTiming(t *testing.T) { + now := time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC) + rows := []ScopedMessage{ + {Role: "user", LocalTime: now, HasLocalTime: true, ContentLength: 12}, + {Role: "assistant", HasLocalTime: false, ContentLength: 0}, + } + got := ScopeTiming(rows) + assert.Equal(t, []TimingMessage{ + {Role: "user", Time: now, Valid: true, ContentLength: 12}, + {Role: "assistant", Time: time.Time{}, Valid: false, ContentLength: 0}, + }, got) +} + +func TestScopeTimingRestoresOrdinalOrderAfterReducer(t *testing.T) { + // The reviewer's case: an empty-model user (ord 0) is buffered behind a + // model-bearing user (ord 1) that emits immediately, then the selected + // assistant (ord 2) flushes the buffer. The reducer therefore emits out of + // ordinal order, but velocity pairs each prompt with the following response + // by position, so ScopeTiming must restore conversation order. + f := ScopeFilter{Models: map[string]struct{}{"sonnet": {}}} + t0 := time.Date(2024, 1, 1, 9, 0, 0, 0, time.UTC) + rows := []MessageInput{ + {SessionID: "s1", Ordinal: 0, Role: "user", HasLocalTime: true, LocalTime: t0, ContentLength: 10}, + {SessionID: "s1", Ordinal: 1, Role: "user", Model: "sonnet", HasLocalTime: true, LocalTime: t0.Add(time.Minute), ContentLength: 20}, + {SessionID: "s1", Ordinal: 2, Role: "assistant", Model: "sonnet", HasLocalTime: true, LocalTime: t0.Add(2 * time.Minute), ContentLength: 30}, + } + emitted, err := collectScopeRows(t, f, rows) + require.NoError(t, err) + require.Len(t, emitted, 3) + require.Equal(t, []int{1, 0, 2}, + []int{emitted[0].Ordinal, emitted[1].Ordinal, emitted[2].Ordinal}, + "reducer emits the model-bearing user ahead of the buffered user") + + got := ScopeTiming(emitted) + assert.Equal(t, []TimingMessage{ + {Role: "user", Time: t0, Valid: true, ContentLength: 10}, + {Role: "user", Time: t0.Add(time.Minute), Valid: true, ContentLength: 20}, + {Role: "assistant", Time: t0.Add(2 * time.Minute), Valid: true, ContentLength: 30}, + }, got, "velocity timing restored to ordinal order") +} diff --git a/internal/db/trends.go b/internal/db/trends.go index 647cb3d2c..64066a25d 100644 --- a/internal/db/trends.go +++ b/internal/db/trends.go @@ -68,15 +68,20 @@ func (db *DB) GetTrendsTerms( sessionFilter.To = "" sessionFilter.DayOfWeek = nil sessionFilter.Hour = nil - where, args := sessionFilter.buildWhereWithoutDate() - query := `SELECT m.content, COALESCE(m.timestamp, ''), + sessionFilter.Model = "" + where, args := sessionFilter.buildWhereWithDate("", false, "s.id") + flt := f.messageScopeFilter() + modelFiltering := len(flt.Models) > 0 + query := `SELECT m.session_id, m.ordinal, m.role, m.is_system, + COALESCE(m.model, ''), m.content, COALESCE(m.timestamp, ''), COALESCE(s.started_at, ''), s.created_at FROM sessions s JOIN messages m ON m.session_id = s.id WHERE ` + where + ` AND m.role IN ('user', 'assistant') AND m.is_system = 0 - AND ` + SystemPrefixSQL("m.content", "m.role") + AND ` + SystemPrefixSQL("m.content", "m.role") + ` + ORDER BY m.session_id, m.ordinal` rows, err := db.getReader().QueryContext(ctx, query, args...) if err != nil { @@ -84,37 +89,87 @@ func (db *DB) GetTrendsTerms( } defer rows.Close() - for rows.Next() { - var content, msgTS, startedAt, createdAt string - if err := rows.Scan( - &content, &msgTS, &startedAt, &createdAt, - ); err != nil { - return TrendsTermsResponse{}, fmt.Errorf("scanning trends term row: %w", err) - } - msgTime, ok := trendMessageLocalTime(msgTS, startedAt, createdAt, loc) + type trendRow struct { + sessionID string + role string + isSystem bool + model string + content string + msgTS string + startedAt string + createdAt string + } + processRow := func(row trendRow) { + msgTime, ok := trendMessageLocalTime(row.msgTS, row.startedAt, row.createdAt, loc) if !ok { - continue - } - if f.HasTimeFilter() && !f.matchesTimeFilter(msgTime) { - continue + return } msgDate := msgTime.Format("2006-01-02") if !inDateRange(msgDate, f.From, f.To) { - continue + return } bucketDate := trendBucketDate(msgTime, loc, granularity) bucket, ok := bucketIndex[bucketDate] if !ok { - continue + return } messageCounts[bucket]++ for i, term := range terms { - count := countTrendOccurrences(content, term) + count := countTrendOccurrences(row.content, term) if count > 0 { counts[i][bucket] += count } } } + rowStartedAt := make(map[string]string) + rowCreatedAt := make(map[string]string) + emit := func(m ScopedMessage) { + processRow(trendRow{ + sessionID: m.SessionID, + role: m.Role, + isSystem: m.IsSystem, + content: m.Content, + msgTS: m.Timestamp, + startedAt: rowStartedAt[m.SessionID], + createdAt: rowCreatedAt[m.SessionID], + }) + } + reducer := NewScopeReducer(flt, emit) + + for rows.Next() { + var row trendRow + var ordinal int + if err := rows.Scan( + &row.sessionID, &ordinal, &row.role, &row.isSystem, + &row.model, &row.content, &row.msgTS, &row.startedAt, + &row.createdAt, + ); err != nil { + return TrendsTermsResponse{}, fmt.Errorf("scanning trends term row: %w", err) + } + if !modelFiltering { + msgTime, ok := trendMessageLocalTime(row.msgTS, row.startedAt, row.createdAt, loc) + if ok && flt.MatchesDayHour(msgTime, true) { + processRow(row) + } + continue + } + rowStartedAt[row.sessionID] = row.startedAt + rowCreatedAt[row.sessionID] = row.createdAt + msgTime, has := trendMessageLocalTime(row.msgTS, row.startedAt, row.createdAt, loc) + if err := reducer.Push(MessageInput{ + SessionID: row.sessionID, + Ordinal: ordinal, + Role: row.role, + Model: row.model, + IsSystem: row.isSystem, + Timestamp: row.msgTS, + LocalTime: msgTime, + HasLocalTime: has, + Content: row.content, + }); err != nil { + return TrendsTermsResponse{}, err + } + } if err := rows.Err(); err != nil { return TrendsTermsResponse{}, fmt.Errorf("iterating trends term rows: %w", err) } diff --git a/internal/db/trends_test.go b/internal/db/trends_test.go index 14a5abb03..69494cc7d 100644 --- a/internal/db/trends_test.go +++ b/internal/db/trends_test.go @@ -186,6 +186,86 @@ func TestGetTrendsTermsSQLiteProjectFilter(t *testing.T) { "project-filtered total") } +func TestGetTrendsTermsSQLiteModelFilter(t *testing.T) { + d := testDB(t) + ctx := context.Background() + start := "2024-06-01T09:00:00Z" + insertSession(t, d, "s1", "proj-a", func(s *Session) { + s.StartedAt = &start + s.CreatedAt = start + s.MessageCount = 1 + s.UserMessageCount = 1 + }) + insertSession(t, d, "s2", "proj-a", func(s *Session) { + s.StartedAt = &start + s.CreatedAt = start + s.MessageCount = 1 + s.UserMessageCount = 1 + }) + insertMessages(t, d, + Message{ + SessionID: "s1", Ordinal: 0, Role: "user", + Content: "seam", Timestamp: start, ContentLength: 4, + Model: "gpt-4o", + }, + Message{ + SessionID: "s2", Ordinal: 0, Role: "user", + Content: "seam", Timestamp: start, ContentLength: 4, + Model: "claude-3-5-sonnet", + }, + ) + terms, err := ParseTrendTerms([]string{"seam"}) + require.NoError(t, err) + got, err := d.GetTrendsTerms(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, terms, "day") + require.NoError(t, err, "GetTrendsTerms") + assert.Equal(t, 1, got.MessageCount, "message count") + assert.Equal(t, 1, trendSeriesByTerm(got.Series)["seam"].Total, + "model-filtered total") +} + +func TestGetTrendsTermsSQLiteModelFilterStaysOnMatchingMessages( + t *testing.T, +) { + d := testDB(t) + ctx := context.Background() + start := "2024-06-01T09:00:00Z" + insertSession(t, d, "s1", "proj-a", func(s *Session) { + s.StartedAt = &start + s.CreatedAt = start + s.MessageCount = 3 + s.UserMessageCount = 1 + }) + insertMessages(t, d, + Message{ + SessionID: "s1", Ordinal: 0, Role: "user", + Content: "seam", Timestamp: start, ContentLength: 4, + }, + Message{ + SessionID: "s1", Ordinal: 1, Role: "assistant", + Content: "ready", Timestamp: start, ContentLength: 5, + Model: "gpt-4o", + }, + Message{ + SessionID: "s1", Ordinal: 2, Role: "assistant", + Content: "seam seam", Timestamp: start, ContentLength: 9, + Model: "claude-3-5-sonnet", + }, + ) + terms, err := ParseTrendTerms([]string{"seam"}) + require.NoError(t, err) + got, err := d.GetTrendsTerms(ctx, AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, terms, "day") + require.NoError(t, err, "GetTrendsTerms") + assert.Equal(t, 2, got.MessageCount, "message count") + assert.Equal(t, 1, trendSeriesByTerm(got.Series)["seam"].Total, + "model-filtered total") +} + func TestGetTrendsTermsSQLiteUsesMessageTimestampRange(t *testing.T) { d := testDB(t) ctx := context.Background() diff --git a/internal/duckdb/analytics_scope.go b/internal/duckdb/analytics_scope.go new file mode 100644 index 000000000..8f29fd083 --- /dev/null +++ b/internal/duckdb/analytics_scope.go @@ -0,0 +1,153 @@ +package duckdb + +import ( + "context" + "fmt" + "strings" + + "go.kenn.io/agentsview/internal/db" +) + +// messageScopeFilter adapts the model/day/hour parts of an AnalyticsFilter into +// the pure db.ScopeFilter. It is a free function because AnalyticsFilter +// is defined in package db. +func messageScopeFilter(f db.AnalyticsFilter) db.ScopeFilter { + models := make(map[string]struct{}) + for _, m := range duckAnalyticsCSVValues(f.Model) { + models[m] = struct{}{} + } + return db.ScopeFilter{ + Models: models, + DayOfWeek: f.DayOfWeek, + Hour: f.Hour, + } +} + +// messageScope holds the matched messages for a model-filtered analytics +// request, grouped by session. It is a pure value; all DB work happens during +// resolution. Its consumers are message stats, velocity timing, and signal +// evidence (which reads message content), mirroring the SQLite/Postgres scopes. +type messageScope struct { + bySession map[string][]db.ScopedMessage +} + +// resolveAnalyticsMessageScope streams candidate messages for sessionIDs and +// reduces them to the model/time-matched set. It returns nil when no model +// filter is set, signalling the caller to keep its session-grain path. +func (s *Store) resolveAnalyticsMessageScope( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, + includeContent bool, +) (*messageScope, error) { + if strings.TrimSpace(f.Model) == "" { + return nil, nil + } + + seen := make(map[string]struct{}, len(sessionIDs)) + unique := make([]string, 0, len(sessionIDs)) + for _, id := range sessionIDs { + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + unique = append(unique, id) + } + + flt := messageScopeFilter(f) + loc := analyticsLocation(f.Timezone) + bySession := make(map[string][]db.ScopedMessage, len(unique)) + emit := func(m db.ScopedMessage) { + bySession[m.SessionID] = append(bySession[m.SessionID], m) + } + + contentExpr := "''" + if includeContent { + contentExpr = "COALESCE(content, '')" + } + + if err := duckQueryChunked(unique, func(chunk []string) error { + reducer := db.NewScopeReducer(flt, emit) + ph, args := duckInPlaceholders(chunk) + rows, err := s.duck.QueryContext(ctx, ` + SELECT session_id, ordinal, role, is_system, COALESCE(model, ''), + has_thinking, has_tool_use, timestamp, + output_tokens, has_output_tokens, content_length, `+contentExpr+` + FROM messages + WHERE session_id IN `+ph+` + ORDER BY session_id, ordinal`, + args..., + ) + if err != nil { + return fmt.Errorf("querying duckdb analytics candidate messages: %w", err) + } + defer rows.Close() + + for rows.Next() { + var ( + sessionID, role, model, content string + ordinal, outputTokens, contentLength int + isSystem, hasThinking, hasToolUse, hasOutputTokens bool + ts any + ) + if err := rows.Scan( + &sessionID, &ordinal, &role, &isSystem, &model, + &hasThinking, &hasToolUse, &ts, &outputTokens, + &hasOutputTokens, &contentLength, &content, + ); err != nil { + return fmt.Errorf("scanning duckdb analytics candidate message: %w", err) + } + tsStr := formatDBTime(ts) + parsed, has := duckLocalTime(tsStr, loc) + if err := reducer.Push(db.MessageInput{ + SessionID: sessionID, + Ordinal: ordinal, + Role: role, + Model: model, + IsSystem: isSystem, + Timestamp: tsStr, + LocalTime: parsed, + HasLocalTime: has, + HasThinking: hasThinking, + HasToolUse: hasToolUse, + OutputTokens: outputTokens, + HasOutputTokens: hasOutputTokens, + ContentLength: contentLength, + Content: content, + }); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating duckdb analytics candidate messages: %w", err) + } + return nil + }); err != nil { + return nil, err + } + + return &messageScope{bySession: bySession}, nil +} + +// MessagesBySession returns the matched rows per session. +func (s *messageScope) MessagesBySession() map[string][]db.ScopedMessage { + return s.bySession +} + +// StatsBySession aggregates matched rows per session. +func (s *messageScope) StatsBySession() map[string]db.MessageStats { + out := make(map[string]db.MessageStats, len(s.bySession)) + for id, rows := range s.bySession { + out[id] = db.ScopeStats(rows) + } + return out +} + +// TimingBySession projects matched rows into the velocity timing view. +func (s *messageScope) TimingBySession() map[string][]db.TimingMessage { + out := make(map[string][]db.TimingMessage, len(s.bySession)) + for id, rows := range s.bySession { + out[id] = db.ScopeTiming(rows) + } + return out +} diff --git a/internal/duckdb/analytics_scope_test.go b/internal/duckdb/analytics_scope_test.go new file mode 100644 index 000000000..b592f1494 --- /dev/null +++ b/internal/duckdb/analytics_scope_test.go @@ -0,0 +1,127 @@ +package duckdb + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" +) + +func TestResolveAnalyticsMessageScope(t *testing.T) { + const ( + sessionA = "duck-scope-a" + sessionB = "duck-scope-b" + model = "claude-3-5-sonnet" + otherM = "gpt-4o" + ts = "2024-06-03T09:00:00Z" // Monday 09:00 UTC + ) + + setup := func(t *testing.T) *Store { + t.Helper() + // sessionA: user turn then SELECTED-model assistant turn (pairs). + // sessionB: user turn then NON-selected-model assistant turn (drops). + return newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession(sessionA, "alpha", "u", ts, 2), + Messages: []db.Message{ + // Empty model so the reducer buffers this user turn and + // flushes it when the selected-model assistant arrives. + duckModelMessage(sessionA, 0, "user", "hello", ts, ""), + duckModelMessage(sessionA, 1, "assistant", "world", ts, model), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + { + Session: syncSession(sessionB, "alpha", "u", ts, 2), + Messages: []db.Message{ + duckModelMessage(sessionB, 0, "user", "hi", ts, ""), + duckModelMessage(sessionB, 1, "assistant", "there", ts, otherM), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + } + + t.Run("blank model returns nil", func(t *testing.T) { + s := setup(t) + scope, err := s.resolveAnalyticsMessageScope( + context.Background(), []string{sessionA}, db.AnalyticsFilter{}, false) + require.NoError(t, err) + assert.Nil(t, scope) + }) + + t.Run("selected model pairs user+assistant; other model yields none", func(t *testing.T) { + s := setup(t) + scope, err := s.resolveAnalyticsMessageScope( + context.Background(), []string{sessionA, sessionB}, + db.AnalyticsFilter{Model: model}, false) + require.NoError(t, err) + require.NotNil(t, scope) + stats := scope.StatsBySession() + sA, ok := stats[sessionA] + require.True(t, ok) + assert.Equal(t, 1, sA.UserMessages) + assert.Equal(t, 1, sA.AssistantMessages) + assert.Equal(t, 2, sA.Messages) + assert.Zero(t, stats[sessionB].Messages, "non-selected model contributes nothing") + }) + + t.Run("TimingBySession returns one entry per matched row", func(t *testing.T) { + s := setup(t) + scope, err := s.resolveAnalyticsMessageScope( + context.Background(), []string{sessionA}, db.AnalyticsFilter{Model: model}, false) + require.NoError(t, err) + require.NotNil(t, scope) + assert.Len(t, scope.TimingBySession()[sessionA], 2) + }) + + t.Run("deduplicates sessionIDs", func(t *testing.T) { + s := setup(t) + scope, err := s.resolveAnalyticsMessageScope( + context.Background(), []string{sessionA, sessionA}, + db.AnalyticsFilter{Model: model}, false) + require.NoError(t, err) + require.NotNil(t, scope) + assert.Equal(t, 2, scope.StatsBySession()[sessionA].Messages, "dedup keeps exactly 2 rows") + }) + + t.Run("hour filter drops non-matching rows", func(t *testing.T) { + s := setup(t) + h := 14 // rows are at 09:00 UTC + scope, err := s.resolveAnalyticsMessageScope( + context.Background(), []string{sessionA}, + db.AnalyticsFilter{Model: model, Hour: &h, Timezone: "UTC"}, false) + require.NoError(t, err) + require.NotNil(t, scope) + assert.Zero(t, scope.StatsBySession()[sessionA].Messages, "hour 14 drops 09:00 rows") + }) + + t.Run("includeContent=false leaves content empty", func(t *testing.T) { + s := setup(t) + scope, err := s.resolveAnalyticsMessageScope( + context.Background(), []string{sessionA}, db.AnalyticsFilter{Model: model}, false) + require.NoError(t, err) + require.NotNil(t, scope) + rows := scope.MessagesBySession()[sessionA] + require.Len(t, rows, 2) + for _, r := range rows { + assert.Empty(t, r.Content) + } + }) + + t.Run("includeContent=true populates content", func(t *testing.T) { + s := setup(t) + scope, err := s.resolveAnalyticsMessageScope( + context.Background(), []string{sessionA}, db.AnalyticsFilter{Model: model}, true) + require.NoError(t, err) + require.NotNil(t, scope) + rows := scope.MessagesBySession()[sessionA] + require.Len(t, rows, 2) + assert.Equal(t, "hello", rows[0].Content) + assert.Equal(t, "world", rows[1].Content) + }) +} diff --git a/internal/duckdb/analytics_usage.go b/internal/duckdb/analytics_usage.go index 933a6dee4..fcd743b08 100644 --- a/internal/duckdb/analytics_usage.go +++ b/internal/duckdb/analytics_usage.go @@ -65,11 +65,17 @@ func (s *Store) analyticsSessions( // analyticsSessionsFiltered loads candidate sessions, optionally applying // the date and hour/day-of-week predicates at the session level. Skill // analytics passes false for both so those filters can be applied to each -// call's own message timestamp instead. +// call's own message timestamp instead. With a model filter and an active +// hour/dow filter it pairs through the shared scope reducer (see +// analyticsSessionsModelTimeFiltered) so an empty-model user turn at the +// selected hour keeps its session, matching how the model-scoped panels count. func (s *Store) analyticsSessionsFiltered( ctx context.Context, f db.AnalyticsFilter, includeDate, includeTime bool, ) ([]duckAnalyticsSession, error) { + if includeTime && f.HasTimeFilter() && strings.TrimSpace(f.Model) != "" { + return s.analyticsSessionsModelTimeFiltered(ctx, f, includeDate) + } where, args := duckBuildAnalyticsWhere( f, "COALESCE(s.started_at, s.created_at)", "s.", includeDate, includeTime) @@ -124,6 +130,41 @@ func (s *Store) analyticsSessionsFiltered( return out, rows.Err() } +// analyticsSessionsModelTimeFiltered loads the date- and model-scoped sessions +// (without the in-SQL day/hour predicate) and keeps only those with at least +// one scoped message matching the hour/dow filter. Running the shared reducer +// instead of the direct m.model time predicate keeps sessions whose matching +// message is an empty-model user turn paired with the selected-model assistant. +func (s *Store) analyticsSessionsModelTimeFiltered( + ctx context.Context, f db.AnalyticsFilter, includeDate bool, +) ([]duckAnalyticsSession, error) { + sessions, err := s.analyticsSessionsFiltered(ctx, f, includeDate, false) + if err != nil { + return nil, err + } + candidateIDs := make([]string, 0, len(sessions)) + for _, session := range sessions { + candidateIDs = append(candidateIDs, session.id) + } + scope, err := s.resolveAnalyticsMessageScope(ctx, candidateIDs, f, false) + if err != nil { + return nil, err + } + matched := make(map[string]struct{}) + if scope != nil { + for id := range scope.MessagesBySession() { + matched[id] = struct{}{} + } + } + out := make([]duckAnalyticsSession, 0, len(sessions)) + for _, session := range sessions { + if _, ok := matched[session.id]; ok { + out = append(out, session) + } + } + return out, nil +} + func duckBuildAnalyticsWhere( f db.AnalyticsFilter, dateCol string, @@ -172,6 +213,11 @@ func duckBuildAnalyticsWhere( if f.Agent != "" { preds, args = appendDuckAnalyticsCSVFilter(preds, args, q("agent"), f.Agent) } + if modelPred, modelArgs := duckAnalyticsCSVPredicate("m.model", f.Model); modelPred != "" { + preds = append(preds, + "EXISTS (SELECT 1 FROM messages m WHERE m.session_id = "+q("id")+" AND "+modelPred+")") + args = append(args, modelArgs...) + } if f.MinUserMessages > 0 { preds = append(preds, q("user_message_count")+" >= ?") args = append(args, f.MinUserMessages) @@ -256,21 +302,11 @@ func duckAutomatedScopePredicate(scope, col string) string { func appendDuckAnalyticsCSVFilter( preds []string, args []any, col string, raw string, ) ([]string, []any) { - values := duckAnalyticsCSVValues(raw) - if len(values) == 0 { - return preds, args - } - if len(values) == 1 { - preds = append(preds, col+" = ?") - args = append(args, values[0]) - return preds, args - } - placeholders := make([]string, len(values)) - for i, value := range values { - placeholders[i] = "?" - args = append(args, value) + pred, predArgs := duckAnalyticsCSVPredicate(col, raw) + if pred != "" { + preds = append(preds, pred) + args = append(args, predArgs...) } - preds = append(preds, col+" IN ("+strings.Join(placeholders, ",")+")") return preds, args } @@ -286,6 +322,25 @@ func duckAnalyticsCSVValues(raw string) []string { return out } +func duckAnalyticsCSVPredicate( + col string, raw string, +) (string, []any) { + values := duckAnalyticsCSVValues(raw) + if len(values) == 0 { + return "", nil + } + if len(values) == 1 { + return col + " = ?", []any{values[0]} + } + placeholders := make([]string, len(values)) + args := make([]any, 0, len(values)) + for i, value := range values { + placeholders[i] = "?" + args = append(args, value) + } + return col + " IN (" + strings.Join(placeholders, ",") + ")", args +} + func duckAnalyticsLocalDateExpr( tsExpr string, f db.AnalyticsFilter, ) (string, []any) { @@ -313,6 +368,10 @@ func duckAnalyticsMessageTimeExists( "m.timestamp IS NOT NULL", } var args []any + if modelPred, modelArgs := duckAnalyticsCSVPredicate("m.model", f.Model); modelPred != "" { + preds = append(preds, modelPred) + args = append(args, modelArgs...) + } if f.DayOfWeek != nil { local, localArgs := duckAnalyticsLocalTimeExpr("m.timestamp", f) preds = append(preds, @@ -450,11 +509,241 @@ func firstNonEmpty(values ...string) string { func round1(v float64) float64 { return math.Round(v*10) / 10 } +func (s *Store) getAnalyticsModelsForSessionIDs( + ctx context.Context, sessionIDs []string, +) ([]string, error) { + if len(sessionIDs) == 0 { + return []string{}, nil + } + models := map[string]bool{} + err := duckQueryChunked(sessionIDs, func(chunk []string) error { + ph, args := duckInPlaceholders(chunk) + rows, err := s.duck.QueryContext(ctx, ` + SELECT DISTINCT model + FROM messages + WHERE session_id IN `+ph+` + AND COALESCE(model, '') <> '' + ORDER BY model`, args...) + if err != nil { + return fmt.Errorf("querying duckdb analytics models: %w", err) + } + defer rows.Close() + for rows.Next() { + var model string + if err := rows.Scan(&model); err != nil { + return fmt.Errorf("scanning duckdb analytics model: %w", err) + } + models[model] = true + } + return rows.Err() + }) + if err != nil { + return nil, err + } + return sortedBoolKeys(models), nil +} + +func (s *Store) getAnalyticsModelsForSessionIDsFiltered( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, +) ([]string, error) { + if len(sessionIDs) == 0 { + return []string{}, nil + } + seen := make(map[string]struct{}, len(sessionIDs)) + unique := make([]string, 0, len(sessionIDs)) + for _, sessionID := range sessionIDs { + if _, ok := seen[sessionID]; ok { + continue + } + seen[sessionID] = struct{}{} + unique = append(unique, sessionID) + } + + filterModels := duckAnalyticsCSVValues(f.Model) + allowedModels := make(map[string]struct{}, len(filterModels)) + for _, model := range filterModels { + allowedModels[model] = struct{}{} + } + loc := analyticsLocation(f.Timezone) + models := map[string]bool{} + err := duckQueryChunked(unique, func(chunk []string) error { + ph, args := duckInPlaceholders(chunk) + rows, err := s.duck.QueryContext(ctx, ` + SELECT model, timestamp + FROM messages + WHERE session_id IN `+ph+` + AND COALESCE(model, '') <> ''`, args...) + if err != nil { + return fmt.Errorf("querying duckdb filtered analytics models: %w", err) + } + defer rows.Close() + for rows.Next() { + var model string + var ts any + if err := rows.Scan(&model, &ts); err != nil { + return fmt.Errorf("scanning duckdb filtered analytics model: %w", err) + } + if len(allowedModels) > 0 { + if _, ok := allowedModels[model]; !ok { + continue + } + } + if f.HasTimeFilter() { + t, ok := parseAnalyticsTime(formatDBTime(ts)) + if !ok || !duckAnalyticsTimeMatches(t.In(loc), f) { + continue + } + } + models[model] = true + } + return rows.Err() + }) + if err != nil { + return nil, err + } + return sortedBoolKeys(models), nil +} + +func (s *Store) getAnalyticsFilteredMessageStats( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, +) (map[string]db.MessageStats, error) { + scope, err := s.resolveAnalyticsMessageScope(ctx, sessionIDs, f, false) + if err != nil { + return nil, err + } + if scope == nil { + return map[string]db.MessageStats{}, nil + } + return scope.StatsBySession(), nil +} + +func (s *Store) analyticsSessionsWithModelMessageCounts( + ctx context.Context, f db.AnalyticsFilter, +) ([]duckAnalyticsSession, error) { + sessions, err := s.analyticsSessions(ctx, f) + if err != nil || strings.TrimSpace(f.Model) == "" || len(sessions) == 0 { + return sessions, err + } + + sessionIDs := make([]string, 0, len(sessions)) + for _, session := range sessions { + sessionIDs = append(sessionIDs, session.id) + } + stats, err := s.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return nil, err + } + for i := range sessions { + stat := stats[sessions[i].id] + sessions[i].messageCount = stat.Messages + sessions[i].totalOutputTokens = stat.OutputTokens + sessions[i].hasTotalOutputTokens = stat.HasOutputTokens + } + return sessions, nil +} + +func (s *Store) getAnalyticsSummaryWithModelCounts( + ctx context.Context, f db.AnalyticsFilter, +) (db.AnalyticsSummary, error) { + sessions, err := s.analyticsSessionsWithModelMessageCounts(ctx, f) + if err != nil { + return db.AnalyticsSummary{}, err + } + + resp := db.AnalyticsSummary{ + Agents: map[string]*db.AgentSummary{}, + Models: []string{}, + } + if len(sessions) == 0 { + return resp, nil + } + + days := map[string]bool{} + projects := map[string]int{} + msgCounts := make([]int, 0, len(sessions)) + sessionIDs := make([]string, 0, len(sessions)) + + for _, session := range sessions { + date := analyticsLocalDate(analyticsDateTime(session), f.Timezone) + resp.TotalSessions++ + resp.TotalMessages += session.messageCount + if session.hasTotalOutputTokens { + resp.TotalOutputTokens += session.totalOutputTokens + resp.TokenReportingSessions++ + } + days[date] = true + projects[session.project] += session.messageCount + msgCounts = append(msgCounts, session.messageCount) + sessionIDs = append(sessionIDs, session.id) + + if resp.Agents[session.agent] == nil { + resp.Agents[session.agent] = &db.AgentSummary{} + } + resp.Agents[session.agent].Sessions++ + resp.Agents[session.agent].Messages += session.messageCount + } + + var models []string + if strings.TrimSpace(f.Model) != "" { + models, err = s.getAnalyticsModelsForSessionIDsFiltered( + ctx, sessionIDs, f, + ) + } else { + models, err = s.getAnalyticsModelsForSessionIDs(ctx, sessionIDs) + } + if err != nil { + return db.AnalyticsSummary{}, err + } + resp.Models = models + resp.ActiveProjects = len(projects) + resp.ActiveDays = len(days) + resp.AvgMessages = round1(float64(resp.TotalMessages) / float64(resp.TotalSessions)) + + sort.Ints(msgCounts) + resp.MedianMessages = median(msgCounts) + if n := len(msgCounts); n > 0 { + resp.P90Messages = msgCounts[min(int(math.Floor(float64(n)*0.9))+1, n)-1] + } + + maxMsgs := -1 + for _, name := range sortedKeys(projects) { + if projects[name] > maxMsgs { + maxMsgs = projects[name] + resp.MostActive = name + } + } + + if resp.TotalMessages > 0 { + counts := make([]int, 0, len(projects)) + for _, count := range projects { + counts = append(counts, count) + } + sort.Sort(sort.Reverse(sort.IntSlice(counts))) + topSum := 0 + for _, count := range counts[:min(3, len(counts))] { + topSum += count + } + resp.Concentration = math.Round( + float64(topSum)/float64(resp.TotalMessages)*1000, + ) / 1000 + } + return resp, nil +} + func (s *Store) GetAnalyticsSummary( ctx context.Context, f db.AnalyticsFilter, ) (db.AnalyticsSummary, error) { // Sum/count aggregate: count subagent sessions (mirrors SQLite). f.IncludeSubagents = true + if strings.TrimSpace(f.Model) != "" { + return s.getAnalyticsSummaryWithModelCounts(ctx, f) + } where, args := duckBuildAnalyticsWhere( f, "COALESCE(s.started_at, s.created_at)", "s.", true, true) localDate, localDateArgs := duckAnalyticsLocalDateExpr( @@ -579,15 +868,164 @@ func (s *Store) GetAnalyticsSummary( if err := agentRows.Err(); err != nil { return db.AnalyticsSummary{}, fmt.Errorf("iterating duckdb analytics summary agents: %w", err) } + sessions, err := s.analyticsSessions(ctx, f) + if err != nil { + return db.AnalyticsSummary{}, err + } + sessionIDs := make([]string, 0, len(sessions)) + for _, sess := range sessions { + sessionIDs = append(sessionIDs, sess.id) + } + var models []string + if f.HasTimeFilter() { + models, err = s.getAnalyticsModelsForSessionIDsFiltered( + ctx, sessionIDs, f, + ) + } else { + models, err = s.getAnalyticsModelsForSessionIDs( + ctx, sessionIDs, + ) + } + if err != nil { + return db.AnalyticsSummary{}, err + } + resp.Models = models return resp, nil } +func (s *Store) getAnalyticsFilteredToolCallCounts( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, +) (map[string]int, error) { + counts := make(map[string]int, len(sessionIDs)) + if len(sessionIDs) == 0 || strings.TrimSpace(f.Model) == "" { + return counts, nil + } + + allowedModels := make(map[string]struct{}) + for _, model := range duckAnalyticsCSVValues(f.Model) { + allowedModels[model] = struct{}{} + } + loc := analyticsLocation(f.Timezone) + err := duckQueryChunked(sessionIDs, func(chunk []string) error { + ph, args := duckInPlaceholders(chunk) + rows, err := s.duck.QueryContext(ctx, ` + SELECT tc.session_id, m.model, m.timestamp, COUNT(*) + FROM tool_calls tc + JOIN messages m + ON m.session_id = tc.session_id + AND m.id = tc.message_id + WHERE tc.session_id IN `+ph+` + GROUP BY tc.session_id, m.model, m.timestamp`, args...) + if err != nil { + return fmt.Errorf( + "querying duckdb filtered analytics tool calls: %w", + err, + ) + } + defer rows.Close() + + for rows.Next() { + var sessionID, model string + var ts any + var count int + if err := rows.Scan(&sessionID, &model, &ts, &count); err != nil { + return fmt.Errorf( + "scanning duckdb filtered analytics tool calls: %w", + err, + ) + } + if _, ok := allowedModels[model]; !ok { + continue + } + if f.HasTimeFilter() { + t, ok := parseAnalyticsTime(formatDBTime(ts)) + if !ok || !duckAnalyticsTimeMatches(t.In(loc), f) { + continue + } + } + counts[sessionID] += count + } + return rows.Err() + }) + if err != nil { + return nil, err + } + + return counts, nil +} + +func (s *Store) getAnalyticsActivityFilteredByModelTime( + ctx context.Context, f db.AnalyticsFilter, granularity string, +) (db.ActivityResponse, error) { + sessions, err := s.analyticsSessions(ctx, f) + if err != nil { + return db.ActivityResponse{}, err + } + sessionIDs := make([]string, 0, len(sessions)) + for _, session := range sessions { + sessionIDs = append(sessionIDs, session.id) + } + messageStats, err := s.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return db.ActivityResponse{}, err + } + toolCounts, err := s.getAnalyticsFilteredToolCallCounts( + ctx, sessionIDs, f, + ) + if err != nil { + return db.ActivityResponse{}, err + } + + out := db.ActivityResponse{Granularity: granularity} + buckets := map[string]*db.ActivityEntry{} + for _, session := range sessions { + date := bucketAnalyticsDate( + analyticsLocalDate(analyticsDateTime(session), f.Timezone), + granularity, + ) + entry := buckets[date] + if entry == nil { + entry = &db.ActivityEntry{ + Date: date, + ByAgent: map[string]int{}, + } + buckets[date] = entry + } + entry.Sessions++ + stat := messageStats[session.id] + entry.Messages += stat.Messages + entry.UserMessages += stat.UserMessages + entry.AssistantMessages += stat.AssistantMessages + entry.ThinkingMessages += stat.ThinkingMessages + entry.ToolCalls += toolCounts[session.id] + entry.ByAgent[session.agent] += stat.Messages + } + + for _, key := range sortedKeys(buckets) { + entry := buckets[key] + if entry == nil { + continue + } + out.Series = append(out.Series, *entry) + } + return out, nil +} + func (s *Store) GetAnalyticsActivity( ctx context.Context, f db.AnalyticsFilter, granularity string, ) (db.ActivityResponse, error) { if granularity == "" { granularity = "day" } + if strings.TrimSpace(f.Model) != "" { + return s.getAnalyticsActivityFilteredByModelTime( + ctx, f, granularity, + ) + } buckets, err := s.queryActivityBuckets(ctx, f, granularity) if err != nil { return db.ActivityResponse{}, err @@ -617,6 +1055,10 @@ func (s *Store) queryActivityBuckets( bucketExpr := duckAnalyticsBucketExpr("local_date", granularity) queryArgs := append([]any{}, localDateArgs...) queryArgs = append(queryArgs, args...) + if _, modelArgs := duckAnalyticsCSVPredicate("m.model", f.Model); len(modelArgs) > 0 { + queryArgs = append(queryArgs, modelArgs...) + queryArgs = append(queryArgs, modelArgs...) + } rows, err := s.duck.QueryContext(ctx, ` WITH filtered_sessions AS ( SELECT s.id, s.message_count, `+localDate+` AS local_date @@ -625,29 +1067,32 @@ func (s *Store) queryActivityBuckets( ), session_rows AS ( SELECT `+bucketExpr+` AS bucket, - COUNT(*) AS sessions, - COALESCE(SUM(message_count), 0) AS messages + COUNT(*) AS sessions FROM filtered_sessions GROUP BY bucket ), message_rows AS ( SELECT `+bucketExpr+` AS bucket, + COUNT(*) AS messages, COUNT(*) FILTER (WHERE m.role = 'user' AND m.is_system = FALSE) AS user_messages, COUNT(*) FILTER (WHERE m.role = 'assistant') AS assistant_messages, COUNT(*) FILTER (WHERE m.has_thinking = TRUE) AS thinking_messages FROM filtered_sessions fs JOIN messages m ON m.session_id = fs.id + `+duckAnalyticsMessageFilterClause("m.model", f.Model)+` GROUP BY bucket ), tool_rows AS ( SELECT `+bucketExpr+` AS bucket, COUNT(*) AS tool_calls FROM filtered_sessions fs JOIN tool_calls tc ON tc.session_id = fs.id + `+duckAnalyticsToolMessageJoin("tc", f.Model)+` + `+duckAnalyticsMessageFilterClause("m.model", f.Model)+` GROUP BY bucket ) SELECT COALESCE(sr.bucket, mr.bucket, tr.bucket) AS bucket, COALESCE(sr.sessions, 0) AS sessions, - COALESCE(sr.messages, 0) AS messages, + COALESCE(mr.messages, 0) AS messages, COALESCE(mr.user_messages, 0) AS user_messages, COALESCE(mr.assistant_messages, 0) AS assistant_messages, COALESCE(mr.thinking_messages, 0) AS thinking_messages, @@ -696,6 +1141,9 @@ func (s *Store) addActivityAgentCounts( bucketExpr := duckAnalyticsBucketExpr("local_date", granularity) queryArgs := append([]any{}, localDateArgs...) queryArgs = append(queryArgs, args...) + if _, modelArgs := duckAnalyticsCSVPredicate("m.model", f.Model); len(modelArgs) > 0 { + queryArgs = append(queryArgs, modelArgs...) + } rows, err := s.duck.QueryContext(ctx, ` WITH filtered_sessions AS ( SELECT s.id, s.agent, `+localDate+` AS local_date @@ -705,6 +1153,7 @@ func (s *Store) addActivityAgentCounts( SELECT `+bucketExpr+` AS bucket, fs.agent, COUNT(*) AS messages FROM filtered_sessions fs JOIN messages m ON m.session_id = fs.id + `+duckAnalyticsMessageFilterClause("m.model", f.Model)+` GROUP BY bucket, fs.agent ORDER BY bucket, fs.agent`, queryArgs..., @@ -768,12 +1217,82 @@ func sortedKeys[V any](m map[string]V) []string { return keys } +func duckAnalyticsMessageFilterClause(col, raw string) string { + pred, _ := duckAnalyticsCSVPredicate(col, raw) + if pred == "" { + return "" + } + return "WHERE " + pred +} + +func duckAnalyticsAndClause(pred string) string { + if pred == "" { + return "" + } + return " AND " + pred +} + +func duckAnalyticsToolMessageJoin( + toolAlias string, model string, +) string { + if model == "" { + return "" + } + return ` + JOIN messages m + ON m.session_id = ` + toolAlias + `.session_id + AND m.id = ` + toolAlias + `.message_id` +} + func (s *Store) GetAnalyticsHeatmap( ctx context.Context, f db.AnalyticsFilter, metric string, ) (db.HeatmapResponse, error) { if metric == "" { metric = "messages" } + if strings.TrimSpace(f.Model) != "" && + (metric == "messages" || metric == "output_tokens" || + metric == "sessions") { + sessions, err := s.analyticsSessionsWithModelMessageCounts(ctx, f) + if err != nil { + return db.HeatmapResponse{}, err + } + counts := map[string]int{} + for _, session := range sessions { + date := analyticsLocalDate(analyticsDateTime(session), f.Timezone) + switch metric { + case "sessions": + counts[date]++ + case "output_tokens": + if session.hasTotalOutputTokens { + counts[date] += session.totalOutputTokens + } + default: + counts[date] += session.messageCount + } + } + entriesFrom := duckClampHeatmapFrom(f.From, f.To) + values := []int{} + for date, v := range counts { + if v > 0 && date >= entriesFrom && date <= f.To { + values = append(values, v) + } + } + sort.Ints(values) + levels := duckComputeHeatmapLevels(values) + entries := duckBuildHeatmapEntries(entriesFrom, f.To, counts, levels) + if metric == "output_tokens" && len(counts) == 0 { + return db.HeatmapResponse{ + Metric: metric, + EntriesFrom: entriesFrom, + }, nil + } + return db.HeatmapResponse{ + Metric: metric, Entries: entries, + Levels: levels, + EntriesFrom: entriesFrom, + }, nil + } where, args := duckBuildAnalyticsWhere( f, "COALESCE(s.started_at, s.created_at)", "s.", true, true) localDate, localDateArgs := duckAnalyticsLocalDateExpr( @@ -911,7 +1430,7 @@ func (s *Store) GetAnalyticsProjects( ) (db.ProjectsAnalyticsResponse, error) { // Per-project aggregate: count subagent sessions (mirrors SQLite). f.IncludeSubagents = true - sessions, err := s.analyticsSessions(ctx, f) + sessions, err := s.analyticsSessionsWithModelMessageCounts(ctx, f) if err != nil { return db.ProjectsAnalyticsResponse{}, err } @@ -969,6 +1488,9 @@ func (s *Store) GetAnalyticsProjects( func (s *Store) GetAnalyticsHourOfWeek( ctx context.Context, f db.AnalyticsFilter, ) (db.HourOfWeekResponse, error) { + if strings.TrimSpace(f.Model) != "" { + return s.getAnalyticsHourOfWeekFilteredByModel(ctx, f) + } sessionFilter := f sessionFilter.DayOfWeek = nil sessionFilter.Hour = nil @@ -1016,15 +1538,52 @@ func (s *Store) GetAnalyticsHourOfWeek( if err := rows.Err(); err != nil { return db.HourOfWeekResponse{}, fmt.Errorf("iterating duckdb analytics hour-of-week: %w", err) } - resp := db.HourOfWeekResponse{Cells: make([]db.HourOfWeekCell, 0, 168)} - for d := range 7 { - for h := range 24 { - resp.Cells = append(resp.Cells, db.HourOfWeekCell{ - DayOfWeek: d, Hour: h, Messages: grid[d][h], - }) + return db.HourOfWeekResponseFromGrid(grid), nil +} + +// getAnalyticsHourOfWeekFilteredByModel buckets model-scoped messages by +// day-of-week and hour. It pairs empty-model user turns with their +// selected-model assistant via the shared scope reducer, so those turns appear +// in the heatmap consistently with the summary, activity, velocity, and trends +// panels. The heatmap is the control that sets the day/hour filter, so it +// clears DayOfWeek/Hour before scoping to keep showing the full grid, matching +// the no-model path. +func (s *Store) getAnalyticsHourOfWeekFilteredByModel( + ctx context.Context, f db.AnalyticsFilter, +) (db.HourOfWeekResponse, error) { + sessions, err := s.analyticsSessionsFiltered(ctx, f, true, false) + if err != nil { + return db.HourOfWeekResponse{}, err + } + sessionIDs := make([]string, 0, len(sessions)) + for _, session := range sessions { + sessionIDs = append(sessionIDs, session.id) + } + + scopeFilter := f + scopeFilter.DayOfWeek = nil + scopeFilter.Hour = nil + scope, err := s.resolveAnalyticsMessageScope( + ctx, sessionIDs, scopeFilter, false, + ) + if err != nil { + return db.HourOfWeekResponse{}, err + } + + var grid [7][24]int + if scope != nil { + for _, msgs := range scope.MessagesBySession() { + for _, m := range msgs { + if !m.HasLocalTime { + continue + } + dow := (int(m.LocalTime.Weekday()) + 6) % 7 + grid[dow][m.LocalTime.Hour()]++ + } } } - return resp, nil + + return db.HourOfWeekResponseFromGrid(grid), nil } func (s *Store) GetAnalyticsSessionShape( @@ -1034,21 +1593,42 @@ func (s *Store) GetAnalyticsSessionShape( if err != nil { return db.SessionShapeResponse{}, err } + modelFilter := strings.TrimSpace(f.Model) != "" lengths := map[string]int{} durations := map[string]int{} ids := []string{} for _, r := range sessions { - lengths[lengthBucket(r.messageCount)]++ ids = append(ids, r.id) + if !modelFilter { + lengths[lengthBucket(r.messageCount)]++ + } if start, okS := parseAnalyticsTime(r.startedAt); okS { if end, okE := parseAnalyticsTime(r.endedAt); okE && !end.Before(start) { durations[durationBucket(end.Sub(start).Minutes())]++ } } } - autonomy, err := s.analyticsAutonomyBuckets(ctx, ids) - if err != nil { - return db.SessionShapeResponse{}, err + autonomy := map[string]int{} + if modelFilter && len(ids) > 0 { + stats, err := s.getAnalyticsFilteredMessageStats(ctx, ids, f) + if err != nil { + return db.SessionShapeResponse{}, err + } + lengths = map[string]int{} + for _, r := range sessions { + stat := stats[r.id] + lengths[lengthBucket(stat.Messages)]++ + if stat.UserMessages > 0 { + ratio := float64(stat.ToolUseMessages) / + float64(stat.UserMessages) + autonomy[autonomyBucket(ratio)]++ + } + } + } else { + autonomy, err = s.analyticsAutonomyBuckets(ctx, ids) + if err != nil { + return db.SessionShapeResponse{}, err + } } return db.SessionShapeResponse{ Count: len(sessions), @@ -1198,7 +1778,9 @@ func duckQueryChunked(ids []string, fn func(chunk []string) error) error { func (s *Store) GetAnalyticsTools( ctx context.Context, f db.AnalyticsFilter, ) (db.ToolsAnalyticsResponse, error) { - sessions, err := s.analyticsSessions(ctx, f) + sessions, err := s.analyticsSessionsFiltered( + ctx, f, false, false, + ) if err != nil { return db.ToolsAnalyticsResponse{}, err } @@ -1215,44 +1797,81 @@ func (s *Store) GetAnalyticsTools( agents := map[string]map[string]int{} trends := map[string]map[string]int{} total := 0 + type toolRow struct { + sessionID string + category string + count int + date string + } + var toolRows []toolRow err = duckQueryChunked(ids, func(chunk []string) error { ph, args := duckInPlaceholders(chunk) - rows, qErr := s.duck.QueryContext(ctx, - `SELECT session_id, category, COUNT(*) - FROM tool_calls - WHERE session_id IN `+ph+` - GROUP BY session_id, category`, args...) + modelPred, modelArgs := duckAnalyticsCSVPredicate("m.model", f.Model) + args = append(args, modelArgs...) + query := `SELECT tc.session_id, tc.category, COUNT(*), + m.timestamp + FROM tool_calls tc + LEFT JOIN messages m + ON m.session_id = tc.session_id + AND m.id = tc.message_id + WHERE tc.session_id IN ` + ph + if modelPred != "" { + query += ` + AND ` + modelPred + } + query += ` + GROUP BY tc.session_id, tc.category, m.timestamp` + rows, qErr := s.duck.QueryContext(ctx, query, args...) if qErr != nil { return qErr } defer rows.Close() for rows.Next() { var sid, cat string + var ts any var count int - if err := rows.Scan(&sid, &cat, &count); err != nil { + if err := rows.Scan(&sid, &cat, &count, &ts); err != nil { return err } r, ok := meta[sid] if !ok { continue } - total += count - cats[cat] += count - if agents[r.agent] == nil { - agents[r.agent] = map[string]int{} - } - agents[r.agent][cat] += count - week := bucketAnalyticsDate(analyticsLocalDate(analyticsDateTime(r), f.Timezone), "week") - if trends[week] == nil { - trends[week] = map[string]int{} + _, date, keep := f.ResolveSkillRowTime( + formatDBTime(ts), analyticsDateTime(r), + ) + if !keep { + continue } - trends[week][cat] += count + toolRows = append(toolRows, toolRow{ + sessionID: sid, + category: cat, + count: count, + date: date, + }) } return rows.Err() }) if err != nil { return db.ToolsAnalyticsResponse{}, err } + for _, tr := range toolRows { + r, ok := meta[tr.sessionID] + if !ok { + continue + } + total += tr.count + cats[tr.category] += tr.count + if agents[r.agent] == nil { + agents[r.agent] = map[string]int{} + } + agents[r.agent][tr.category] += tr.count + week := bucketAnalyticsDate(tr.date, "week") + if trends[week] == nil { + trends[week] = map[string]int{} + } + trends[week][tr.category] += tr.count + } resp := db.ToolsAnalyticsResponse{TotalCalls: total} for cat, count := range cats { resp.ByCategory = append(resp.ByCategory, db.ToolCategoryCount{ @@ -1302,6 +1921,8 @@ func (s *Store) GetAnalyticsSkills( var skillRows []db.SkillAnalyticsRow err = duckQueryChunked(ids, func(chunk []string) error { ph, args := duckInPlaceholders(chunk) + modelPred, modelArgs := duckAnalyticsCSVPredicate("m.model", f.Model) + args = append(args, modelArgs...) rows, qErr := s.duck.QueryContext(ctx, `SELECT tc.session_id, TRIM(COALESCE(tc.skill_name, '')), COUNT(*), m.timestamp @@ -1311,6 +1932,7 @@ func (s *Store) GetAnalyticsSkills( AND m.id = tc.message_id WHERE tc.session_id IN `+ph+` AND TRIM(COALESCE(tc.skill_name, '')) != '' + `+duckAnalyticsAndClause(modelPred)+` GROUP BY tc.session_id, TRIM(COALESCE(tc.skill_name, '')), m.timestamp`, args...) if qErr != nil { @@ -1375,12 +1997,41 @@ func (s *Store) GetAnalyticsVelocity( mc: sess.messageCount, } } + if strings.TrimSpace(f.Model) != "" { + stats, err := s.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return db.VelocityResponse{}, err + } + for _, sid := range sessionIDs { + info := sessionInfo[sid] + info.mc = stats[sid].Messages + sessionInfo[sid] = info + } + } - sessionMsgs, err := s.velocityMessages(ctx, sessionIDs, analyticsLocation(f.Timezone)) + var sessionMsgs map[string][]duckVelocityMsg + if strings.TrimSpace(f.Model) != "" { + sessionMsgs, err = s.filteredVelocityMessages( + ctx, sessionIDs, f, + ) + } else { + sessionMsgs, err = s.velocityMessages( + ctx, sessionIDs, analyticsLocation(f.Timezone), + ) + } if err != nil { return db.VelocityResponse{}, err } - toolCounts, err := s.velocityToolCounts(ctx, sessionIDs) + var toolCounts map[string]int + if strings.TrimSpace(f.Model) != "" { + toolCounts, err = s.getAnalyticsFilteredToolCallCounts( + ctx, sessionIDs, f, + ) + } else { + toolCounts, err = s.velocityToolCounts(ctx, sessionIDs) + } if err != nil { return db.VelocityResponse{}, err } @@ -1507,6 +2158,36 @@ func (s *Store) velocityMessages( return out, rows.Err() } +func (s *Store) filteredVelocityMessages( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, +) (map[string][]duckVelocityMsg, error) { + out := make(map[string][]duckVelocityMsg, len(sessionIDs)) + if len(sessionIDs) == 0 { + return out, nil + } + + scope, err := s.resolveAnalyticsMessageScope(ctx, sessionIDs, f, false) + if err != nil { + return nil, err + } + if scope == nil { + return out, nil + } + for sessionID, rows := range scope.TimingBySession() { + for _, row := range rows { + out[sessionID] = append(out[sessionID], duckVelocityMsg{ + role: row.Role, + ts: row.Time, + valid: row.Valid, + contentLength: row.ContentLength, + }) + } + } + return out, nil +} + func (s *Store) velocityToolCounts( ctx context.Context, sessionIDs []string, @@ -1693,8 +2374,76 @@ func (s *Store) GetAnalyticsTopSessions( metric = "messages" } + if strings.TrimSpace(f.Model) != "" && + (metric == "messages" || metric == "output_tokens") { + sessions, err := s.analyticsSessionsWithModelMessageCounts(ctx, f) + if err != nil { + return db.TopSessionsResponse{}, err + } + sort.SliceStable(sessions, func(i, j int) bool { + if metric == "output_tokens" { + if sessions[i].totalOutputTokens != sessions[j].totalOutputTokens { + return sessions[i].totalOutputTokens > + sessions[j].totalOutputTokens + } + } else if sessions[i].messageCount != sessions[j].messageCount { + return sessions[i].messageCount > + sessions[j].messageCount + } + return sessions[i].id < sessions[j].id + }) + + out := db.TopSessionsResponse{Metric: metric} + for i := range sessions { + if metric == "output_tokens" && + !sessions[i].hasTotalOutputTokens { + continue + } + if len(out.Sessions) >= 10 { + break + } + startedAt := sessions[i].startedAt + endedAt := sessions[i].endedAt + out.Sessions = append(out.Sessions, db.TopSession{ + ID: sessions[i].id, + Project: sessions[i].project, + FirstMessage: sessions[i].firstMessage, + DisplayName: sessions[i].displayName, + MessageCount: sessions[i].messageCount, + OutputTokens: sessions[i].totalOutputTokens, + DurationMin: duckSessionDurationMinutes(sessions[i]), + StartedAt: &startedAt, + EndedAt: &endedAt, + TerminationStatus: sessions[i].terminationStatus, + }) + } + return out, nil + } + + includeTime := true + var pairedSet map[string]bool + if f.HasTimeFilter() && strings.TrimSpace(f.Model) != "" { + // Filter the scoped session set in Go rather than binding every + // paired ID into one IN (...) predicate, which would exceed the + // driver bind-variable cap for large result sets. Mirrors the + // SQLite/PostgreSQL top-sessions Go path under a model filter: load + // the model+date candidates, then keep only the paired sessions and + // limit in Go. The in-SQL ORDER BY still ranks them by the metric. + paired, err := s.analyticsSessionsModelTimeFiltered(ctx, f, true) + if err != nil { + return db.TopSessionsResponse{}, err + } + if len(paired) == 0 { + return db.TopSessionsResponse{Metric: metric}, nil + } + pairedSet = make(map[string]bool, len(paired)) + for _, session := range paired { + pairedSet[session.id] = true + } + includeTime = false + } where, args := duckBuildAnalyticsWhere( - f, "COALESCE(s.started_at, s.created_at)", "s.", true, true) + f, "COALESCE(s.started_at, s.created_at)", "s.", true, includeTime) durationExpr := "(epoch(s.ended_at) - epoch(s.started_at)) / 60.0" durationSelectExpr := "COALESCE(" + durationExpr + ", 0)" activeDurationExpr := fmt.Sprintf(` @@ -1726,6 +2475,13 @@ func (s *Store) GetAnalyticsTopSessions( where += " AND s.has_total_output_tokens = TRUE" orderExpr = "s.total_output_tokens DESC, s.id ASC" } + // When filtering the scoped set in Go (model+time), drop the SQL LIMIT so + // the paired sessions aren't truncated before the Go filter; the in-SQL + // ORDER BY keeps them ranked and the top 10 is taken after filtering. + limitClause := "\n\t\tLIMIT 10" + if pairedSet != nil { + limitClause = "" + } query := ` SELECT s.id, s.project, s.first_message, s.message_count, s.total_output_tokens, ` + durationSelectExpr + ` AS duration_min, @@ -1733,8 +2489,7 @@ func (s *Store) GetAnalyticsTopSessions( s.started_at, s.ended_at, s.termination_status FROM sessions s WHERE ` + where + ` - ORDER BY ` + orderExpr + ` - LIMIT 10` + ORDER BY ` + orderExpr + limitClause rows, err := s.duck.QueryContext(ctx, query, args...) if err != nil { return db.TopSessionsResponse{}, fmt.Errorf("querying duckdb analytics top sessions: %w", err) @@ -1753,6 +2508,9 @@ func (s *Store) GetAnalyticsTopSessions( ); err != nil { return db.TopSessionsResponse{}, fmt.Errorf("scanning duckdb analytics top session: %w", err) } + if pairedSet != nil && !pairedSet[row.ID] { + continue + } startedAt := formatDBTime(startedRaw) endedAt := formatDBTime(endedRaw) row.StartedAt = &startedAt @@ -1760,6 +2518,9 @@ func (s *Store) GetAnalyticsTopSessions( row.DurationMin = round1(row.DurationMin) row.ActiveDurationMin = round1(row.ActiveDurationMin) out.Sessions = append(out.Sessions, row) + if pairedSet != nil && len(out.Sessions) >= 10 { + break + } } if err := rows.Err(); err != nil { return db.TopSessionsResponse{}, fmt.Errorf("iterating duckdb analytics top sessions: %w", err) @@ -1767,6 +2528,19 @@ func (s *Store) GetAnalyticsTopSessions( return out, nil } +func duckSessionDurationMinutes(session duckAnalyticsSession) float64 { + startedAt, okStart := parseAnalyticsTime(session.startedAt) + endedAt, okEnd := parseAnalyticsTime(session.endedAt) + if !okStart || !okEnd || endedAt.Before(startedAt) { + return 0 + } + return round1(endedAt.Sub(startedAt).Minutes()) +} + +// GetAnalyticsSignals returns aggregated session signal data. Signals stay +// session-scoped under a model filter (totals are session-level aggregates +// over sessions that used the model, not re-attributed per model); see the +// SQLite GetAnalyticsSignals for the rationale. func (s *Store) GetAnalyticsSignals( ctx context.Context, f db.AnalyticsFilter, ) (db.SignalsAnalyticsResponse, error) { @@ -1802,7 +2576,7 @@ func (s *Store) GetAnalyticsSignalSessions( return db.SignalSessionsResponse{}, err } candidates := db.SignalCandidates(rows, signal, limit) - messages, err := s.duckSignalMessages(ctx, candidates) + messages, err := s.duckSignalMessages(ctx, candidates, f) if err != nil { return db.SignalSessionsResponse{}, err } @@ -1898,21 +2672,61 @@ func (s *Store) duckPopulateFrustrationMarkers( func (s *Store) duckSignalMessages( ctx context.Context, rows []db.SignalRow, + f db.AnalyticsFilter, ) (map[string][]db.SignalMessage, error) { out := make(map[string][]db.SignalMessage, len(rows)) if len(rows) == 0 { return out, nil } + if strings.TrimSpace(f.Model) != "" { + ids := make([]string, 0, len(rows)) + for _, r := range rows { + ids = append(ids, r.ID) + } + scope, err := s.resolveAnalyticsMessageScope(ctx, ids, f, true) + if err != nil { + return nil, err + } + if scope != nil { + for sessionID, scopedRows := range scope.MessagesBySession() { + for _, row := range scopedRows { + out[sessionID] = append(out[sessionID], db.SignalMessage{ + SessionID: row.SessionID, + Ordinal: row.Ordinal, + Role: row.Role, + Content: row.Content, + Timestamp: row.Timestamp, + IsSystem: row.IsSystem, + HasToolUse: row.HasToolUse, + }) + } + } + } + return out, nil + } placeholders := make([]string, len(rows)) args := make([]any, len(rows)) for i, r := range rows { placeholders[i] = "?" args[i] = r.ID } + filterModels := duckAnalyticsCSVValues(f.Model) q := `SELECT session_id, ordinal, role, content, timestamp, is_system, has_tool_use FROM messages - WHERE session_id IN (` + strings.Join(placeholders, ",") + `) + WHERE session_id IN (` + strings.Join(placeholders, ",") + `)` + if len(filterModels) == 1 { + q += ` AND model = ?` + args = append(args, filterModels[0]) + } else if len(filterModels) > 1 { + modelPlaceholders := make([]string, len(filterModels)) + for i, model := range filterModels { + modelPlaceholders[i] = "?" + args = append(args, model) + } + q += ` AND model IN (` + strings.Join(modelPlaceholders, ",") + `)` + } + q += ` ORDER BY session_id, ordinal` msgRows, err := s.duck.QueryContext(ctx, q, args...) if err != nil { @@ -1958,6 +2772,7 @@ func (s *Store) GetTrendsTerms( sessionFilter := f sessionFilter.From = "" sessionFilter.To = "" + sessionFilter.Model = "" sessionFilter.DayOfWeek = nil sessionFilter.Hour = nil sessions, err := s.analyticsSessions(ctx, sessionFilter) @@ -1973,54 +2788,96 @@ func (s *Store) GetTrendsTerms( f.From, f.To, granularity, buckets, terms, counts, messageCounts, ), nil } + loc := analyticsLocation(f.Timezone) + flt := messageScopeFilter(f) + modelFiltering := len(flt.Models) > 0 + trendLocal := func(msgTS, startedAt, createdAt any) (time.Time, bool) { + ts := firstNonEmpty(formatDBTime(msgTS), formatDBTime(startedAt), formatDBTime(createdAt)) + t, ok := parseAnalyticsTime(ts) + if !ok { + return time.Time{}, false + } + return t.In(loc), true + } rows, err := s.duck.QueryContext(ctx, ` - SELECT m.session_id, m.content, m.timestamp, s.started_at, s.created_at + SELECT m.session_id, m.ordinal, m.role, m.is_system, + COALESCE(m.model, ''), m.content, m.timestamp, + s.started_at, s.created_at FROM messages m JOIN sessions s ON s.id = m.session_id WHERE s.deleted_at IS NULL AND m.role IN ('user', 'assistant') AND m.is_system = FALSE - AND `+db.DuckDBSystemPrefixSQL("m.content", "m.role")) + AND `+db.DuckDBSystemPrefixSQL("m.content", "m.role")+` + ORDER BY m.session_id, m.ordinal`) if err != nil { return db.TrendsTermsResponse{}, err } defer rows.Close() - for rows.Next() { - var sessionID string - var content string - var msgTS, startedAt, createdAt any - if err := rows.Scan(&sessionID, &content, &msgTS, &startedAt, &createdAt); err != nil { - return db.TrendsTermsResponse{}, err - } + type trendRow struct { + sessionID string + role string + isSystem bool + model string + content string + msgTS any + startedAt any + createdAt any + } + processRow := func(sessionID, content string, local time.Time) { if !allowedSessions[sessionID] { - continue - } - ts := firstNonEmpty(formatDBTime(msgTS), formatDBTime(startedAt), formatDBTime(createdAt)) - t, ok := parseAnalyticsTime(ts) - if !ok { - continue - } - local := t.In(analyticsLocation(f.Timezone)) - if !duckAnalyticsTimeMatches(local, f) { - continue + return } date := local.Format("2006-01-02") if f.From != "" && date < f.From { - continue + return } if f.To != "" && date > f.To { - continue + return } bucket := bucketAnalyticsDate(date, granularity) pos, ok := index[bucket] if !ok { - continue + return } messageCounts[pos]++ for i, term := range terms { counts[i][pos] += db.CountTrendOccurrences(content, term) } } + emit := func(m db.ScopedMessage) { + if !m.HasLocalTime { + return + } + processRow(m.SessionID, m.Content, m.LocalTime) + } + reducer := db.NewScopeReducer(flt, emit) + for rows.Next() { + var row trendRow + var ordinal int + if err := rows.Scan(&row.sessionID, &ordinal, &row.role, &row.isSystem, &row.model, &row.content, &row.msgTS, &row.startedAt, &row.createdAt); err != nil { + return db.TrendsTermsResponse{}, err + } + local, has := trendLocal(row.msgTS, row.startedAt, row.createdAt) + if !modelFiltering { + if has && flt.MatchesDayHour(local, true) { + processRow(row.sessionID, row.content, local) + } + continue + } + if err := reducer.Push(db.MessageInput{ + SessionID: row.sessionID, + Ordinal: ordinal, + Role: row.role, + Model: row.model, + IsSystem: row.isSystem, + LocalTime: local, + HasLocalTime: has, + Content: row.content, + }); err != nil { + return db.TrendsTermsResponse{}, err + } + } if err := rows.Err(); err != nil { return db.TrendsTermsResponse{}, err } diff --git a/internal/duckdb/analytics_usage_test.go b/internal/duckdb/analytics_usage_test.go index 62e3b5fdc..8da307c15 100644 --- a/internal/duckdb/analytics_usage_test.go +++ b/internal/duckdb/analytics_usage_test.go @@ -2,6 +2,8 @@ package duckdb import ( "context" + "fmt" + "path/filepath" "strings" "testing" @@ -117,6 +119,35 @@ func TestDuckAnalyticsAutomatedScopeOneShotExemption(t *testing.T) { "DuckDB analytics SQL missing one-shot exemption") } +func TestDuckAnalyticsModelFilterPredicates(t *testing.T) { + sql, _ := duckBuildAnalyticsWhere( + db.AnalyticsFilter{Model: "gpt-4o"}, + "COALESCE(s.started_at, s.created_at)", + "s.", + false, + false, + ) + assert.Contains(t, sql, + "EXISTS (SELECT 1 FROM messages m WHERE m.session_id = s.id AND m.model = ?)") +} + +func TestDuckAnalyticsModelAndHourUseSameMessagePredicate(t *testing.T) { + hour := 10 + sql, _ := duckBuildAnalyticsWhere( + db.AnalyticsFilter{ + Model: "gpt-4o", + Hour: &hour, + }, + "COALESCE(s.started_at, s.created_at)", + "s.", + false, + true, + ) + assert.Contains(t, sql, "m.session_id = s.id") + assert.Contains(t, sql, "m.model = ?") + assert.Contains(t, sql, "CAST(strftime(") +} + func TestDuckUsageTerminationPredicate(t *testing.T) { where, args := appendDuckUsageSessionFilterClauses( "WHERE true", @@ -216,9 +247,1309 @@ func TestDuckSignalMessagesFormatsTimestampValues(t *testing.T) { NULL, FALSE, FALSE)`) require.NoError(t, err) - got, err := store.duckSignalMessages(ctx, []db.SignalRow{{ID: "signal-time"}}) + got, err := store.duckSignalMessages( + ctx, + []db.SignalRow{{ID: "signal-time"}}, + db.AnalyticsFilter{}, + ) require.NoError(t, err) require.Len(t, got["signal-time"], 2) assert.Equal(t, "2026-01-20T12:34:56Z", got["signal-time"][0].Timestamp) assert.Empty(t, got["signal-time"][1].Timestamp) } + +func TestDuckAnalyticsSignalSessionsModelFilterUsesMatchingMessages( + t *testing.T, +) { + ctx := context.Background() + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-signal-mixed", "alpha", "mixed", + "2024-06-01T09:00:00Z", 2, + ), + Messages: []db.Message{ + duckModelMessage( + "duck-signal-mixed", 0, "assistant", + "claude tool evidence", + "2024-06-01T09:05:00Z", + "claude-3-5-sonnet", + db.ToolCall{ToolName: "Grep", Category: "Grep"}, + ), + duckModelMessage( + "duck-signal-mixed", 1, "assistant", + "gpt tool evidence", + "2024-06-01T09:06:00Z", + "gpt-4o", + db.ToolCall{ToolName: "Read", Category: "Read"}, + ), + }, + Signals: db.SessionSignalUpdate{ + ToolFailureSignalCount: 1, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + resp, err := store.GetAnalyticsSignalSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "tool_failure_signals", 10) + require.NoError(t, err, "GetAnalyticsSignalSessions") + require.Len(t, resp.Sessions, 1, "len(Sessions)") + assert.Equal(t, "gpt tool evidence", resp.Sessions[0].Excerpt) + require.NotNil(t, resp.Sessions[0].MessageOrdinal) + assert.Equal(t, 1, *resp.Sessions[0].MessageOrdinal) +} + +func TestDuckAnalyticsSignalSessionsModelFilterKeepsParserUserEvidence( + t *testing.T, +) { + ctx := context.Background() + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-signal-parser-user", "alpha", "mixed", + "2024-06-01T09:00:00Z", 2, + ), + Messages: []db.Message{ + duckModelMessage( + "duck-signal-parser-user", 0, "user", + "help", "2024-06-01T09:00:00Z", "", + ), + duckModelMessage( + "duck-signal-parser-user", 1, "assistant", + "reply", "2024-06-01T09:01:00Z", "gpt-4o", + db.ToolCall{ToolName: "Read", Category: "Read"}, + ), + }, + Signals: db.SessionSignalUpdate{ + QualitySignals: db.QualitySignals{ + Version: 1, + ShortPromptCount: 1, + }, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + resp, err := store.GetAnalyticsSignalSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "short_prompt_count", 10) + require.NoError(t, err, "GetAnalyticsSignalSessions") + require.Len(t, resp.Sessions, 1, "len(Sessions)") + assert.Equal(t, "help", resp.Sessions[0].Excerpt) + require.NotNil(t, resp.Sessions[0].MessageOrdinal) + assert.Equal(t, 0, *resp.Sessions[0].MessageOrdinal) +} + +func TestDuckAnalyticsSummaryModelFilterPopulatesModels(t *testing.T) { + ctx := context.Background() + start := "2024-06-01T09:00:00Z" + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession("duck-model-a", "alpha", "gpt", start, 1), + Messages: []db.Message{ + duckModelMessage("duck-model-a", 0, "assistant", "gpt", start, "gpt-4o"), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + { + Session: syncSession("duck-model-b", "alpha", "claude", start, 1), + Messages: []db.Message{ + duckModelMessage("duck-model-b", 0, "assistant", "claude", start, "claude-3-5-sonnet"), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + resp, err := store.GetAnalyticsSummary(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsSummary") + assert.Equal(t, 1, resp.TotalSessions, "TotalSessions") + assert.Equal(t, []string{"gpt-4o"}, resp.Models, "Models") +} + +func TestDuckAnalyticsSummaryModelFilterCountsOnlyMatchingMessages(t *testing.T) { + ctx := context.Background() + store := newDuckMixedModelAnalyticsStore(t) + + resp, err := store.GetAnalyticsSummary(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsSummary") + assert.Equal(t, 1, resp.TotalSessions, "TotalSessions") + assert.Equal(t, 1, resp.TotalMessages, "TotalMessages") + assert.Equal(t, []string{"gpt-4o"}, resp.Models, "Models") + assert.Equal(t, 1.0, resp.AvgMessages, "AvgMessages") + assert.Equal(t, 1, resp.MedianMessages, "MedianMessages") + assert.Equal(t, 1, resp.P90Messages, "P90Messages") + require.Len(t, resp.Agents, 1, "len(Agents)") + for _, summary := range resp.Agents { + assert.Equal(t, 1, summary.Messages, "AgentMessages") + } +} + +func TestDuckAnalyticsSummaryModelsUseMatchingHourRowsOnly(t *testing.T) { + ctx := context.Background() + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-summary-hour-mixed", "alpha", "mixed", + "2024-06-01T09:00:00Z", 2, + ), + Messages: []db.Message{ + duckModelMessage( + "duck-summary-hour-mixed", 0, "assistant", "gpt", + "2024-06-01T09:05:00Z", "gpt-4o", + ), + duckModelMessage( + "duck-summary-hour-mixed", 1, "assistant", "claude", + "2024-06-01T10:05:00Z", "claude-3-5-sonnet", + ), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + hour := 9 + resp, err := store.GetAnalyticsSummary(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Hour: &hour, + }) + require.NoError(t, err, "GetAnalyticsSummary") + assert.Equal(t, 1, resp.TotalSessions, "TotalSessions") + assert.Equal(t, []string{"gpt-4o"}, resp.Models, "Models") +} + +func TestDuckAnalyticsSummaryModelFilterUsesFilteredOutputTokens( + t *testing.T, +) { + ctx := context.Background() + + mixedSession := syncSession( + "duck-summary-output-mixed", "alpha", "mixed", + "2024-06-01T10:00:00Z", 2, + ) + mixedSession.TotalOutputTokens = 111 + mixedSession.HasTotalOutputTokens = true + mixedGpt := duckModelMessage( + "duck-summary-output-mixed", 0, "assistant", "gpt", + "2024-06-01T10:00:00Z", "gpt-4o", + ) + mixedGpt.TokenUsage = []byte(`{"output_tokens":11}`) + mixedGpt.OutputTokens = 11 + mixedGpt.HasOutputTokens = true + mixedClaude := duckModelMessage( + "duck-summary-output-mixed", 1, "assistant", "claude", + "2024-06-01T10:05:00Z", "claude-3-5-sonnet", + ) + mixedClaude.TokenUsage = []byte(`{"output_tokens":100}`) + mixedClaude.OutputTokens = 100 + mixedClaude.HasOutputTokens = true + + uncoveredSession := syncSession( + "duck-summary-output-uncovered", "alpha", "mixed", + "2024-06-01T10:40:00Z", 2, + ) + uncoveredSession.TotalOutputTokens = 90 + uncoveredSession.HasTotalOutputTokens = true + uncoveredGpt := duckModelMessage( + "duck-summary-output-uncovered", 0, "assistant", "gpt", + "2024-06-01T10:40:00Z", "gpt-4o", + ) + uncoveredGpt.TokenUsage = nil + uncoveredGpt.ContextTokens = 0 + uncoveredGpt.OutputTokens = 0 + uncoveredGpt.HasContextTokens = false + uncoveredGpt.HasOutputTokens = false + uncoveredClaude := duckModelMessage( + "duck-summary-output-uncovered", 1, "assistant", "claude", + "2024-06-01T10:45:00Z", "claude-3-5-sonnet", + ) + uncoveredClaude.TokenUsage = []byte(`{"output_tokens":90}`) + uncoveredClaude.OutputTokens = 90 + uncoveredClaude.HasOutputTokens = true + + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: mixedSession, + Messages: []db.Message{ + mixedGpt, + mixedClaude, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + { + Session: uncoveredSession, + Messages: []db.Message{ + uncoveredGpt, + uncoveredClaude, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + hour := 10 + resp, err := store.GetAnalyticsSummary(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }) + require.NoError(t, err, "GetAnalyticsSummary") + assert.Equal(t, 2, resp.TotalSessions, "TotalSessions") + assert.Equal(t, 2, resp.TotalMessages, "TotalMessages") + assert.Equal(t, []string{"gpt-4o"}, resp.Models, "Models") + assert.Equal(t, 11, resp.TotalOutputTokens, "TotalOutputTokens") + assert.Equal(t, 1, resp.TokenReportingSessions, "TokenReportingSessions") +} + +func TestDuckAnalyticsActivityModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + ctx := context.Background() + store := newDuckMixedModelAnalyticsStore(t) + + resp, err := store.GetAnalyticsActivity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "day") + require.NoError(t, err, "GetAnalyticsActivity") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 1, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].AssistantMessages, + "AssistantMessages") + assert.Equal(t, 2, resp.Series[0].ToolCalls, "ToolCalls") +} + +func TestDuckAnalyticsActivityModelAndHourFilterCountsOnlyMatchingHourRows( + t *testing.T, +) { + ctx := context.Background() + + readMsg := duckModelMessage( + "duck-activity-hour-gpt", 0, "assistant", "read", + "2024-06-01T09:00:00Z", "gpt-4o", + db.ToolCall{ + ToolName: "Read", Category: "Read", + }, + db.ToolCall{ + ToolName: "Bash", Category: "Bash", + }, + ) + grepMsg := duckModelMessage( + "duck-activity-hour-gpt", 1, "assistant", "grep", + "2024-06-01T10:00:00Z", "gpt-4o", + db.ToolCall{ + ToolName: "Grep", Category: "Grep", + }, + ) + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-activity-hour-gpt", "alpha", "mixed", + "2024-06-01T09:00:00Z", 2, + ), + Messages: []db.Message{ + readMsg, + grepMsg, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + hour := 10 + resp, err := store.GetAnalyticsActivity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "day") + require.NoError(t, err, "GetAnalyticsActivity") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 1, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].AssistantMessages, + "AssistantMessages") + assert.Equal(t, 1, resp.Series[0].ToolCalls, "ToolCalls") +} + +func TestDuckAnalyticsHourOfWeekModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + ctx := context.Background() + store := newDuckMixedModelAnalyticsStore(t) + + resp, err := store.GetAnalyticsHourOfWeek(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsHourOfWeek") + assert.Equal(t, 1, duckHOWMessages(resp.Cells, 5, 9), "Sat 09:00") + assert.Equal(t, 0, duckHOWMessages(resp.Cells, 5, 10), "Sat 10:00") +} + +func TestDuckAnalyticsHourOfWeekModelFilterIncludesPairedUserTurns( + t *testing.T, +) { + ctx := context.Background() + start := "2024-06-01T09:00:00Z" + reply := "2024-06-01T10:00:00Z" + + userMsg := syncMessage("duck-paired-model", 0, "user", "q", start) + userMsg.Model = "" + assistantMsg := duckModelMessage( + "duck-paired-model", 1, "assistant", "a", reply, "gpt-4o", + ) + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-paired-model", "alpha", "q", start, 2, + ), + Messages: []db.Message{userMsg, assistantMsg}, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + resp, err := store.GetAnalyticsHourOfWeek(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsHourOfWeek") + assert.Equal(t, 1, duckHOWMessages(resp.Cells, 5, 9), + "paired empty-model user turn at Sat 09:00") + assert.Equal(t, 1, duckHOWMessages(resp.Cells, 5, 10), + "selected-model assistant at Sat 10:00") +} + +func TestDuckAnalyticsActivityModelAndHourFilterKeepsPairedUserTurn( + t *testing.T, +) { + ctx := context.Background() + userMsg := syncMessage( + "duck-activity-paired-hour", 0, "user", "q", "2024-06-01T09:00:00Z", + ) + userMsg.Model = "" + assistantMsg := duckModelMessage( + "duck-activity-paired-hour", 1, "assistant", "a", + "2024-06-01T10:00:00Z", "gpt-4o", + ) + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-activity-paired-hour", "alpha", "q", + "2024-06-01T09:00:00Z", 2, + ), + Messages: []db.Message{userMsg, assistantMsg}, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + // Paired empty-model user turn at hour 9; gpt-4o assistant at hour 10. + // Filtering by hour 9 must keep the session via the paired user turn. + hour := 9 + resp, err := store.GetAnalyticsActivity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "day") + require.NoError(t, err, "GetAnalyticsActivity") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 1, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].UserMessages, "UserMessages") + assert.Equal(t, 0, resp.Series[0].AssistantMessages, "AssistantMessages") +} + +func TestDuckAnalyticsHeatmapSessionsModelAndHourFilterKeepsPairedUserTurn( + t *testing.T, +) { + ctx := context.Background() + userMsg := syncMessage( + "duck-heatmap-sessions-paired", 0, "user", "q", + "2024-06-01T09:00:00Z", + ) + userMsg.Model = "" + assistantMsg := duckModelMessage( + "duck-heatmap-sessions-paired", 1, "assistant", "a", + "2024-06-01T10:00:00Z", "gpt-4o", + ) + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-heatmap-sessions-paired", "alpha", "q", + "2024-06-01T09:00:00Z", 2, + ), + Messages: []db.Message{userMsg, assistantMsg}, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + // Empty-model user turn at hour 9 paired with a gpt-4o assistant at hour + // 10. The sessions heatmap must keep the session via the paired user turn + // instead of requiring a gpt-4o message at hour 9. + hour := 9 + resp, err := store.GetAnalyticsHeatmap(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "sessions") + require.NoError(t, err, "GetAnalyticsHeatmap") + require.Len(t, resp.Entries, 1, "len(Entries)") + assert.Equal(t, 1, resp.Entries[0].Value, "Value") +} + +func TestDuckAnalyticsTopSessionsDurationModelAndHourFilterKeepsPairedUserTurn( + t *testing.T, +) { + ctx := context.Background() + session := syncSession( + "duck-top-duration-paired", "alpha", "q", + "2024-06-01T09:00:00Z", 2, + ) + endedAt := "2024-06-01T10:00:00Z" + session.EndedAt = &endedAt + userMsg := syncMessage( + "duck-top-duration-paired", 0, "user", "q", + "2024-06-01T09:00:00Z", + ) + userMsg.Model = "" + assistantMsg := duckModelMessage( + "duck-top-duration-paired", 1, "assistant", "a", + "2024-06-01T10:00:00Z", "gpt-4o", + ) + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: session, + Messages: []db.Message{userMsg, assistantMsg}, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + // Empty-model user turn at hour 9 paired with a gpt-4o assistant at hour + // 10. Ranking sessions by duration under the gpt-4o + hour-9 filter must + // keep the session via the paired user turn. + hour := 9 + resp, err := store.GetAnalyticsTopSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "duration") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 1, "len(Sessions)") + assert.Equal(t, "duck-top-duration-paired", resp.Sessions[0].ID, "ID") +} + +func TestDuckAnalyticsTopSessionsDurationModelFilterRanksAndLimitsScopedSet( + t *testing.T, +) { + ctx := context.Background() + // 12 gpt-4o sessions in hour 9 with distinct active durations (message gap + // k*20s, under the 5-minute cap). The model+hour duration path filters the + // scoped set and limits in Go without binding every ID into one IN list, so + // it must return the 10 longest by active duration, not all 12. + var writes []db.SessionBatchWrite + for k := 1; k <= 12; k++ { + id := fmt.Sprintf("duck-top-dur-rank-%02d", k) + start := "2024-06-01T09:00:00Z" + gap := k * 20 + end := fmt.Sprintf("2024-06-01T09:%02d:%02dZ", gap/60, gap%60) + session := syncSession(id, "alpha", "q", start, 2) + session.EndedAt = &end + userMsg := syncMessage(id, 0, "user", "q", start) + userMsg.Model = "" + writes = append(writes, db.SessionBatchWrite{ + Session: session, + Messages: []db.Message{ + userMsg, + duckModelMessage(id, 1, "assistant", "a", end, "gpt-4o"), + }, + DataVersion: 1, + ReplaceMessages: true, + }) + } + store := newDuckAnalyticsStore(t, writes) + + hour := 9 + resp, err := store.GetAnalyticsTopSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "duration") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 10, "top sessions capped at 10") + // Longest active duration first: k=12 (240s) down to k=3 (60s). + assert.Equal(t, "duck-top-dur-rank-12", resp.Sessions[0].ID, "longest") + assert.Equal(t, "duck-top-dur-rank-03", resp.Sessions[9].ID, "tenth") + ids := map[string]bool{} + for _, session := range resp.Sessions { + ids[session.ID] = true + } + assert.False(t, ids["duck-top-dur-rank-01"], "shortest excluded") + assert.False(t, ids["duck-top-dur-rank-02"], "second shortest excluded") +} + +func TestDuckAnalyticsTopSessionsDurationModelFilterHandlesManyScopedSessions( + t *testing.T, +) { + ctx := context.Background() + // More scoped sessions than duckMaxSQLVars, the cap the rest of the codebase + // chunks IN (...) predicates against. The model+time duration path filters + // and ranks the scoped set in Go instead of binding every ID into one + // predicate, so a set larger than the cap still returns the correct top 10. + const scoped = duckMaxSQLVars + 50 + var writes []db.SessionBatchWrite + for k := range scoped { + id := fmt.Sprintf("duck-top-many-%04d", k) + writes = append(writes, db.SessionBatchWrite{ + Session: syncSession(id, "alpha", "q", "2024-06-01T09:00:00Z", 1), + Messages: []db.Message{ + duckModelMessage(id, 0, "assistant", "a", + "2024-06-01T09:00:00Z", "gpt-4o"), + }, + DataVersion: 1, + ReplaceMessages: true, + }) + } + // One clear outlier with a long active span so ranking is observable. + longID := "duck-top-many-longest" + longSession := syncSession(longID, "alpha", "q", "2024-06-01T09:00:00Z", 2) + longEnd := "2024-06-01T09:04:00Z" + longSession.EndedAt = &longEnd + writes = append(writes, db.SessionBatchWrite{ + Session: longSession, + Messages: []db.Message{ + duckModelMessage(longID, 0, "assistant", "a", + "2024-06-01T09:00:00Z", "gpt-4o"), + duckModelMessage(longID, 1, "assistant", "b", + "2024-06-01T09:04:00Z", "gpt-4o"), + }, + DataVersion: 1, + ReplaceMessages: true, + }) + store := newDuckAnalyticsStore(t, writes) + + hour := 9 + resp, err := store.GetAnalyticsTopSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "duration") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 10, "top sessions capped at 10") + assert.Equal(t, longID, resp.Sessions[0].ID, + "longest active span ranks first across the scoped set") +} + +func TestDuckAnalyticsToolsModelFilterCountsOnlyMatchingToolCalls( + t *testing.T, +) { + ctx := context.Background() + store := newDuckMixedModelAnalyticsStore(t) + + resp, err := store.GetAnalyticsTools(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsTools") + assert.Equal(t, 2, resp.TotalCalls, "TotalCalls") + + byCategory := map[string]int{} + for _, row := range resp.ByCategory { + byCategory[row.Category] = row.Count + } + assert.Equal(t, 1, byCategory["Read"], "Read") + assert.Equal(t, 1, byCategory["Skill"], "Skill") + assert.Zero(t, byCategory["Grep"], "Grep") +} + +func TestDuckAnalyticsToolsModelAndHourFilterCountsOnlyMatchingHourToolCalls( + t *testing.T, +) { + ctx := context.Background() + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-tool-model-hour", "alpha", "mixed", + "2024-06-01T09:00:00Z", 2, + ), + Messages: []db.Message{ + duckModelMessage( + "duck-tool-model-hour", 0, "assistant", + "read", "2024-06-01T09:00:00Z", "gpt-4o", + db.ToolCall{ToolName: "Read", Category: "Read"}, + db.ToolCall{ToolName: "Bash", Category: "Bash"}, + ), + duckModelMessage( + "duck-tool-model-hour", 1, "assistant", + "grep", "2024-06-01T10:00:00Z", "gpt-4o", + db.ToolCall{ToolName: "Grep", Category: "Grep"}, + ), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + hour := 10 + resp, err := store.GetAnalyticsTools(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }) + require.NoError(t, err, "GetAnalyticsTools") + assert.Equal(t, 1, resp.TotalCalls, "TotalCalls") + require.Len(t, resp.ByCategory, 1, "len(ByCategory)") + assert.Equal(t, "Grep", resp.ByCategory[0].Category, "Category") + assert.Equal(t, 1, resp.ByCategory[0].Count, "Count") +} + +func TestDuckAnalyticsSkillsModelFilterCountsOnlyMatchingSkillCalls( + t *testing.T, +) { + ctx := context.Background() + store := newDuckMixedModelAnalyticsStore(t) + + resp, err := store.GetAnalyticsSkills(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsSkills") + assert.Equal(t, 1, resp.TotalSkillCalls, "TotalSkillCalls") + assert.Equal(t, 1, resp.DistinctSkills, "DistinctSkills") + require.Len(t, resp.BySkill, 1, "len(BySkill)") + assert.Equal(t, "review-code", resp.BySkill[0].SkillName, "SkillName") + assert.Equal(t, 1, resp.BySkill[0].CallCount, "CallCount") +} + +func TestDuckAnalyticsProjectsModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + ctx := context.Background() + store := newDuckMixedModelAnalyticsStore(t) + + resp, err := store.GetAnalyticsProjects(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsProjects") + require.Len(t, resp.Projects, 1, "len(Projects)") + assert.Equal(t, 1, resp.Projects[0].Messages, "Messages") + assert.Equal(t, 1.0, resp.Projects[0].AvgMessages, "AvgMessages") + assert.Equal(t, 1, resp.Projects[0].MedianMessages, "MedianMessages") + assert.Equal(t, 1.0, resp.Projects[0].DailyTrend, "DailyTrend") +} + +func TestDuckAnalyticsHeatmapModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + ctx := context.Background() + store := newDuckMixedModelAnalyticsStore(t) + + resp, err := store.GetAnalyticsHeatmap(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "messages") + require.NoError(t, err, "GetAnalyticsHeatmap") + require.Len(t, resp.Entries, 1, "len(Entries)") + assert.Equal(t, 1, resp.Entries[0].Value, "Value") +} + +func TestDuckAnalyticsHeatmapModelFilterUsesFilteredOutputTokens( + t *testing.T, +) { + ctx := context.Background() + + mixedSession := syncSession( + "duck-heatmap-output-mixed", "alpha", "mixed", + "2024-06-01T10:00:00Z", 2, + ) + mixedSession.TotalOutputTokens = 111 + mixedSession.HasTotalOutputTokens = true + mixedGpt := duckModelMessage( + "duck-heatmap-output-mixed", 0, "assistant", "gpt", + "2024-06-01T10:00:00Z", "gpt-4o", + ) + mixedGpt.TokenUsage = []byte(`{"output_tokens":11}`) + mixedGpt.OutputTokens = 11 + mixedGpt.HasOutputTokens = true + mixedClaude := duckModelMessage( + "duck-heatmap-output-mixed", 1, "assistant", "claude", + "2024-06-01T10:05:00Z", "claude-3-5-sonnet", + ) + mixedClaude.TokenUsage = []byte(`{"output_tokens":100}`) + mixedClaude.OutputTokens = 100 + mixedClaude.HasOutputTokens = true + + uncoveredSession := syncSession( + "duck-heatmap-output-uncovered", "alpha", "mixed", + "2024-06-01T10:40:00Z", 2, + ) + uncoveredSession.TotalOutputTokens = 90 + uncoveredSession.HasTotalOutputTokens = true + uncoveredGpt := duckModelMessage( + "duck-heatmap-output-uncovered", 0, "assistant", "gpt", + "2024-06-01T10:40:00Z", "gpt-4o", + ) + uncoveredGpt.TokenUsage = nil + uncoveredGpt.ContextTokens = 0 + uncoveredGpt.OutputTokens = 0 + uncoveredGpt.HasContextTokens = false + uncoveredGpt.HasOutputTokens = false + uncoveredClaude := duckModelMessage( + "duck-heatmap-output-uncovered", 1, "assistant", "claude", + "2024-06-01T10:45:00Z", "claude-3-5-sonnet", + ) + uncoveredClaude.TokenUsage = []byte(`{"output_tokens":90}`) + uncoveredClaude.OutputTokens = 90 + uncoveredClaude.HasOutputTokens = true + + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: mixedSession, + Messages: []db.Message{ + mixedGpt, + mixedClaude, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + { + Session: uncoveredSession, + Messages: []db.Message{ + uncoveredGpt, + uncoveredClaude, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + hour := 10 + resp, err := store.GetAnalyticsHeatmap(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "output_tokens") + require.NoError(t, err, "GetAnalyticsHeatmap") + require.Len(t, resp.Entries, 1, "len(Entries)") + assert.Equal(t, 11, resp.Entries[0].Value, "Value") +} + +func TestDuckAnalyticsTopSessionsMessagesUseFilteredModelCounts( + t *testing.T, +) { + ctx := context.Background() + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-top-mixed", "alpha", "mixed", + "2024-06-01T09:00:00Z", 3, + ), + Messages: []db.Message{ + duckModelMessage( + "duck-top-mixed", 0, "assistant", "gpt", + "2024-06-01T09:00:00Z", "gpt-4o", + ), + duckModelMessage( + "duck-top-mixed", 1, "assistant", "claude", + "2024-06-01T09:05:00Z", "claude-3-5-sonnet", + ), + duckModelMessage( + "duck-top-mixed", 2, "assistant", "claude", + "2024-06-01T09:06:00Z", "claude-3-5-sonnet", + ), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + { + Session: syncSession( + "duck-top-gpt", "alpha", "gpt", + "2024-06-01T11:00:00Z", 2, + ), + Messages: []db.Message{ + duckModelMessage( + "duck-top-gpt", 0, "assistant", "gpt", + "2024-06-01T11:00:00Z", "gpt-4o", + ), + duckModelMessage( + "duck-top-gpt", 1, "assistant", "gpt", + "2024-06-01T11:05:00Z", "gpt-4o", + ), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + resp, err := store.GetAnalyticsTopSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "messages") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 2, "len(Sessions)") + assert.Equal(t, "duck-top-gpt", resp.Sessions[0].ID, "top session") + assert.Equal(t, 2, resp.Sessions[0].MessageCount, "top MessageCount") + assert.Equal(t, "duck-top-mixed", resp.Sessions[1].ID, "second session") + assert.Equal(t, 1, resp.Sessions[1].MessageCount, "second MessageCount") +} + +func TestDuckAnalyticsTopSessionsOutputTokensUseFilteredModelTotals( + t *testing.T, +) { + ctx := context.Background() + + mixedSession := syncSession( + "duck-top-output-mixed", "alpha", "mixed", + "2024-06-01T09:00:00Z", 2, + ) + mixedSession.TotalOutputTokens = 510 + mixedSession.HasTotalOutputTokens = true + mixedGpt := duckModelMessage( + "duck-top-output-mixed", 0, "assistant", "gpt", + "2024-06-01T09:00:00Z", "gpt-4o", + ) + mixedGpt.TokenUsage = []byte(`{"output_tokens":10}`) + mixedGpt.OutputTokens = 10 + mixedGpt.HasOutputTokens = true + mixedClaude := duckModelMessage( + "duck-top-output-mixed", 1, "assistant", "claude", + "2024-06-01T09:05:00Z", "claude-3-5-sonnet", + ) + mixedClaude.TokenUsage = []byte(`{"output_tokens":500}`) + mixedClaude.OutputTokens = 500 + mixedClaude.HasOutputTokens = true + + gptSession := syncSession( + "duck-top-output-gpt", "alpha", "gpt", + "2024-06-01T11:00:00Z", 1, + ) + gptSession.TotalOutputTokens = 30 + gptSession.HasTotalOutputTokens = true + gptMsg := duckModelMessage( + "duck-top-output-gpt", 0, "assistant", "gpt", + "2024-06-01T11:00:00Z", "gpt-4o", + ) + gptMsg.TokenUsage = []byte(`{"output_tokens":30}`) + gptMsg.OutputTokens = 30 + gptMsg.HasOutputTokens = true + + uncoveredSession := syncSession( + "duck-top-output-uncovered", "alpha", "mixed", + "2024-06-01T13:00:00Z", 2, + ) + uncoveredSession.TotalOutputTokens = 900 + uncoveredSession.HasTotalOutputTokens = true + uncoveredGpt := duckModelMessage( + "duck-top-output-uncovered", 0, "assistant", "gpt", + "2024-06-01T13:00:00Z", "gpt-4o", + ) + uncoveredGpt.TokenUsage = nil + uncoveredGpt.ContextTokens = 0 + uncoveredGpt.OutputTokens = 0 + uncoveredGpt.HasContextTokens = false + uncoveredGpt.HasOutputTokens = false + uncoveredClaude := duckModelMessage( + "duck-top-output-uncovered", 1, "assistant", "claude", + "2024-06-01T13:05:00Z", "claude-3-5-sonnet", + ) + uncoveredClaude.TokenUsage = []byte(`{"output_tokens":900}`) + uncoveredClaude.OutputTokens = 900 + uncoveredClaude.HasOutputTokens = true + + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: mixedSession, + Messages: []db.Message{ + mixedGpt, + mixedClaude, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + { + Session: gptSession, + Messages: []db.Message{ + gptMsg, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + { + Session: uncoveredSession, + Messages: []db.Message{ + uncoveredGpt, + uncoveredClaude, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + resp, err := store.GetAnalyticsTopSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "output_tokens") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 2, "len(Sessions)") + assert.Equal(t, "duck-top-output-gpt", resp.Sessions[0].ID, + "top session") + assert.Equal(t, 30, resp.Sessions[0].OutputTokens, + "top OutputTokens") + assert.Equal(t, "duck-top-output-mixed", resp.Sessions[1].ID, + "second session") + assert.Equal(t, 10, resp.Sessions[1].OutputTokens, + "second OutputTokens") +} + +func TestDuckAnalyticsVelocityModelFilterUsesMatchingRowsOnly(t *testing.T) { + ctx := context.Background() + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-velocity-model", "alpha", "mixed", + "2024-06-01T09:00:00Z", 4, + ), + Messages: []db.Message{ + duckModelMessage( + "duck-velocity-model", 0, "user", "claude q", + "2024-06-01T09:00:00Z", "claude-3-5-sonnet", + ), + duckModelMessage( + "duck-velocity-model", 1, "assistant", "offscope-offscope-xx", + "2024-06-01T09:00:10Z", "claude-3-5-sonnet", + db.ToolCall{ToolName: "Read", Category: "Read"}, + db.ToolCall{ToolName: "Bash", Category: "Bash"}, + db.ToolCall{ToolName: "Grep", Category: "Grep"}, + ), + duckModelMessage( + "duck-velocity-model", 2, "user", "gpt q", + "2024-06-01T09:10:00Z", "", + ), + duckModelMessage( + "duck-velocity-model", 3, "assistant", "reply", + "2024-06-01T09:11:00Z", "gpt-4o", + db.ToolCall{ToolName: "Edit", Category: "Edit"}, + db.ToolCall{ToolName: "Write", Category: "Write"}, + ), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + resp, err := store.GetAnalyticsVelocity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsVelocity") + assert.Equal(t, 60.0, resp.Overall.FirstResponseSec.P50, + "FirstResponse P50") + assert.Equal(t, 2.0, resp.Overall.MsgsPerActiveMin, + "MsgsPerActiveMin") + assert.Equal(t, 5.0, resp.Overall.CharsPerActiveMin, + "CharsPerActiveMin") + assert.Equal(t, 2.0, resp.Overall.ToolCallsPerActiveMin, + "ToolCallsPerActiveMin") +} + +func TestDuckAnalyticsVelocityModelFilterCountsNullTimestampToolCallsWithoutTimeFilter( + t *testing.T, +) { + ctx := context.Background() + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-velocity-null-ts", "alpha", "mixed", + "2024-06-01T09:00:00Z", 4, + ), + Messages: []db.Message{ + duckModelMessage( + "duck-velocity-null-ts", 0, "user", "claude q", + "2024-06-01T09:00:00Z", "claude-3-5-sonnet", + ), + duckModelMessage( + "duck-velocity-null-ts", 1, "assistant", "offscope-offscope-xx", + "2024-06-01T09:00:10Z", "claude-3-5-sonnet", + db.ToolCall{ToolName: "Read", Category: "Read"}, + db.ToolCall{ToolName: "Bash", Category: "Bash"}, + db.ToolCall{ToolName: "Grep", Category: "Grep"}, + ), + duckModelMessage( + "duck-velocity-null-ts", 2, "user", "gpt q", + "2024-06-01T09:10:00Z", "", + ), + duckModelMessage( + "duck-velocity-null-ts", 3, "assistant", "reply", + "2024-06-01T09:11:00Z", "gpt-4o", + db.ToolCall{ToolName: "Edit", Category: "Edit"}, + db.ToolCall{ToolName: "Write", Category: "Write"}, + ), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + _, err := store.duck.ExecContext(ctx, ` + UPDATE sessions + SET message_count = 5 + WHERE id = 'duck-velocity-null-ts'`) + require.NoError(t, err, "update session message_count") + _, err = store.duck.ExecContext(ctx, ` + INSERT INTO messages ( + id, session_id, ordinal, role, content, timestamp, + has_tool_use, content_length, is_system, model + ) VALUES + (9103, 'duck-velocity-null-ts', 4, 'assistant', 'extra', + NULL, TRUE, 5, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert null-timestamp message") + _, err = store.duck.ExecContext(ctx, ` + INSERT INTO tool_calls ( + id, message_id, session_id, tool_name, category, call_index + ) VALUES + (9203, 9103, 'duck-velocity-null-ts', 'Search', 'Search', 0)`) + require.NoError(t, err, "insert tool call") + + resp, err := store.GetAnalyticsVelocity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsVelocity") + assert.Equal(t, 60.0, resp.Overall.FirstResponseSec.P50, + "FirstResponse P50") + assert.Equal(t, 3.0, resp.Overall.ToolCallsPerActiveMin, + "ToolCallsPerActiveMin") +} + +func TestDuckAnalyticsSessionShapeModelFilterUsesMatchingRowsOnly(t *testing.T) { + ctx := context.Background() + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-shape-model", "alpha", "mixed", + "2024-06-01T09:00:00Z", 6, + ), + Messages: []db.Message{ + duckModelMessage( + "duck-shape-model", 0, "user", "gpt q", + "2024-06-01T09:00:00Z", "", + ), + duckModelMessage( + "duck-shape-model", 1, "assistant", "gpt tool", + "2024-06-01T09:01:00Z", "gpt-4o", + db.ToolCall{ToolName: "Read", Category: "Read"}, + ), + duckModelMessage( + "duck-shape-model", 2, "user", "claude q1", + "2024-06-01T09:02:00Z", "claude-3-5-sonnet", + ), + duckModelMessage( + "duck-shape-model", 3, "user", "claude q2", + "2024-06-01T09:03:00Z", "claude-3-5-sonnet", + ), + duckModelMessage( + "duck-shape-model", 4, "user", "claude q3", + "2024-06-01T09:04:00Z", "claude-3-5-sonnet", + ), + duckModelMessage( + "duck-shape-model", 5, "assistant", "claude reply", + "2024-06-01T09:05:00Z", "claude-3-5-sonnet", + ), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + resp, err := store.GetAnalyticsSessionShape(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsSessionShape") + assert.Equal(t, 1, resp.Count, "Count") + + lenMap := map[string]int{} + for _, bucket := range resp.LengthDistribution { + lenMap[bucket.Label] = bucket.Count + } + assert.Equal(t, 1, lenMap["1-5"], "filtered length bucket") + assert.Equal(t, 0, lenMap["6-15"], "full-session count must not leak") + + autoMap := map[string]int{} + for _, bucket := range resp.AutonomyDistribution { + autoMap[bucket.Label] = bucket.Count + } + assert.Equal(t, 1, autoMap["1-2"], "filtered autonomy bucket") + assert.Equal(t, 0, autoMap["<0.5"], "off-model user turns must not leak") +} + +func TestDuckAnalyticsVelocityModelFilterUsesMatchingComplexityBucket(t *testing.T) { + ctx := context.Background() + msgs := []db.Message{ + duckModelMessage( + "duck-velocity-complexity", 0, "user", "gpt q", + "2024-06-01T09:00:00Z", "", + ), + duckModelMessage( + "duck-velocity-complexity", 1, "assistant", "reply", + "2024-06-01T09:01:00Z", "gpt-4o", + ), + } + for i := 2; i < 16; i++ { + role := "user" + if i%2 == 1 { + role = "assistant" + } + msgs = append(msgs, duckModelMessage( + "duck-velocity-complexity", i, role, "claude", + "2024-06-01T09:10:00Z", "claude-3-5-sonnet", + )) + } + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-velocity-complexity", "alpha", "mixed", + "2024-06-01T09:00:00Z", 16, + ), + Messages: msgs, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + resp, err := store.GetAnalyticsVelocity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsVelocity") + require.Len(t, resp.ByComplexity, 1, "len(ByComplexity)") + assert.Equal(t, "1-15", resp.ByComplexity[0].Label, + "complexity bucket should use filtered message count") + assert.Equal(t, 1, resp.ByComplexity[0].Sessions, "Sessions") +} + +func TestDuckTrendsTermsModelFilterStaysOnMatchingMessages(t *testing.T) { + ctx := context.Background() + store := newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-trends-model", "alpha", "mixed", + "2024-06-01T09:00:00Z", 3, + ), + Messages: []db.Message{ + duckModelMessage( + "duck-trends-model", 0, "user", "seam", + "2024-06-01T09:00:00Z", "", + ), + duckModelMessage( + "duck-trends-model", 1, "assistant", "ready", + "2024-06-01T09:01:00Z", "gpt-4o", + ), + duckModelMessage( + "duck-trends-model", 2, "assistant", "seam seam", + "2024-06-01T10:00:00Z", "claude-3-5-sonnet", + ), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + terms, err := db.ParseTrendTerms([]string{"seam"}) + require.NoError(t, err) + + resp, err := store.GetTrendsTerms(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, terms, "day") + require.NoError(t, err, "GetTrendsTerms") + assert.Equal(t, 2, resp.MessageCount, "MessageCount") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Total, "Total") +} + +func newDuckAnalyticsStore( + t *testing.T, writes []db.SessionBatchWrite, +) *Store { + t.Helper() + ctx := context.Background() + local := newLocalDB(t) + _, err := local.WriteSessionBatchAtomic(writes) + require.NoError(t, err) + syncer := newTestSync( + t, + filepath.Join(t.TempDir(), "analytics-model.duckdb"), + local, + SyncOptions{}, + ) + _, err = syncer.Push(ctx, true, nil) + require.NoError(t, err) + return NewStoreFromDB(syncer.DB()) +} + +func newDuckMixedModelAnalyticsStore(t *testing.T) *Store { + t.Helper() + startA := "2024-06-01T09:00:00Z" + startB := "2024-06-01T10:00:00Z" + + gptMsg := duckModelMessage( + "duck-mixed-model", 0, "assistant", "seam", startA, "gpt-4o", + db.ToolCall{ + ToolName: "Read", Category: "Read", + }, + db.ToolCall{ + ToolName: "Skill", Category: "Skill", SkillName: "review-code", + }, + ) + claudeMsg := duckModelMessage( + "duck-mixed-model", 1, "assistant", "seam seam", startB, + "claude-3-5-sonnet", + db.ToolCall{ + ToolName: "Grep", Category: "Grep", + }, + db.ToolCall{ + ToolName: "Skill", Category: "Skill", SkillName: "write-tests", + }, + ) + + return newDuckAnalyticsStore(t, []db.SessionBatchWrite{ + { + Session: syncSession( + "duck-mixed-model", "alpha", "mixed", startA, 2, + ), + Messages: []db.Message{ + gptMsg, + claudeMsg, + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) +} + +func duckModelMessage( + sessionID string, + ordinal int, + role, content, ts, model string, + calls ...db.ToolCall, +) db.Message { + msg := syncMessage(sessionID, ordinal, role, content, ts, calls...) + msg.Model = model + return msg +} + +func duckHOWMessages(cells []db.HourOfWeekCell, dow, hour int) int { + for _, cell := range cells { + if cell.DayOfWeek == dow && cell.Hour == hour { + return cell.Messages + } + } + return -1 +} diff --git a/internal/postgres/activityreport.go b/internal/postgres/activityreport.go index 377d297fd..3f76e1648 100644 --- a/internal/postgres/activityreport.go +++ b/internal/postgres/activityreport.go @@ -81,7 +81,7 @@ func (s *Store) activityReportSessions( ctx context.Context, f db.AnalyticsFilter, rangeStartUTC, rangeEndUTC string, ) ([]activity.SessionMeta, []string, error) { pb := ¶mBuilder{} - where := buildAnalyticsWhereWithDate(f, "", pb, false) + where := buildAnalyticsWhereWithDate(f, "", pb, false, "s.id") lower := pb.add(rangeStartUTC) upper := pb.add(rangeEndUTC) diff --git a/internal/postgres/analytics.go b/internal/postgres/analytics.go index a85ce76b9..9b6821061 100644 --- a/internal/postgres/analytics.go +++ b/internal/postgres/analytics.go @@ -98,7 +98,9 @@ func buildAnalyticsWhere( dateCol string, pb *paramBuilder, ) string { - return buildAnalyticsWhereWithDate(f, dateCol, pb, true) + return buildAnalyticsWhereWithDate( + f, dateCol, pb, true, "id", + ) } // buildAnalyticsWhereWithoutDate returns common analytics @@ -109,7 +111,9 @@ func buildAnalyticsWhereWithoutDate( f db.AnalyticsFilter, pb *paramBuilder, ) string { - return buildAnalyticsWhereWithDate(f, "", pb, false) + return buildAnalyticsWhereWithDate( + f, "", pb, false, "id", + ) } func buildAnalyticsWhereWithDate( @@ -117,7 +121,11 @@ func buildAnalyticsWhereWithDate( dateCol string, pb *paramBuilder, includeDate bool, + sessionIDExpr string, ) string { + if sessionIDExpr == "" { + sessionIDExpr = "id" + } preds := []string{ "message_count > 0", // Mirror the SQLite analytics filter: count subagents only on @@ -144,6 +152,26 @@ func buildAnalyticsWhereWithDate( preds = appendPGAnalyticsCSVFilter( preds, "agent", f.Agent, pb) } + if f.Model != "" { + models := csvFilterValues(f.Model) + if len(models) == 1 { + preds = append(preds, + "EXISTS (SELECT 1 FROM messages m WHERE "+ + "m.session_id = "+sessionIDExpr+ + " AND m.model = "+pb.add(models[0])+")") + } else if len(models) > 1 { + phs := make([]string, len(models)) + for i, model := range models { + phs[i] = pb.add(model) + } + preds = append(preds, + "EXISTS (SELECT 1 FROM messages m WHERE "+ + "m.session_id = "+sessionIDExpr+ + " AND m.model IN ("+ + strings.Join(phs, ",")+ + "))") + } + } if f.MinUserMessages > 0 { preds = append(preds, "user_message_count >= "+ @@ -201,6 +229,10 @@ func appendPGAnalyticsCSVFilter( } func pgAnalyticsCSVValues(raw string) []string { + return csvFilterValues(raw) +} + +func csvFilterValues(raw string) []string { values := strings.Split(raw, ",") out := values[:0] for _, value := range values { @@ -212,6 +244,54 @@ func pgAnalyticsCSVValues(raw string) []string { return out } +func (s *Store) getAnalyticsFilteredMessageCounts( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, +) (map[string]int, error) { + stats, err := s.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return nil, err + } + counts := make(map[string]int, len(stats)) + for sessionID, stat := range stats { + counts[sessionID] = stat.Messages + } + return counts, nil +} + +func (s *Store) getAnalyticsModelScopedMessages( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, +) (map[string][]db.ScopedMessage, error) { + scope, err := s.resolveAnalyticsMessageScope(ctx, sessionIDs, f, true) + if err != nil { + return nil, err + } + if scope == nil { + return map[string][]db.ScopedMessage{}, nil + } + return scope.MessagesBySession(), nil +} + +func (s *Store) getAnalyticsFilteredMessageStats( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, +) (map[string]db.MessageStats, error) { + scope, err := s.resolveAnalyticsMessageScope(ctx, sessionIDs, f, false) + if err != nil { + return nil, err + } + if scope == nil { + return map[string]db.MessageStats{}, nil + } + return scope.StatsBySession(), nil +} + // localTime parses a UTC timestamp string and converts it to // the given location. func localTime( @@ -321,19 +401,32 @@ const pgDateColS = "COALESCE(s.started_at, s.created_at)" // filteredSessionIDs returns session IDs that have at least // one message matching the hour/dow filter. +// filteredSessionIDs returns the session IDs that have at least one message +// matching the hour/dow filter. With a model filter active it pairs through the +// shared scope reducer (see filteredSessionIDsModel) so an empty-model user turn +// at the selected hour keeps its session, matching the model-scoped panels. func (s *Store) filteredSessionIDs( ctx context.Context, f db.AnalyticsFilter, ) (map[string]bool, error) { + if strings.TrimSpace(f.Model) != "" { + return s.filteredSessionIDsModel(ctx, f) + } loc := analyticsLocation(f) pb := ¶mBuilder{} - where := buildAnalyticsWhere(f, pgDateColS, pb) + where := buildAnalyticsWhereWithDate( + f, pgDateColS, pb, true, "s.id", + ) + preds := []string{ + where, + "m.timestamp IS NOT NULL", + } query := `SELECT s.id, TO_CHAR(m.timestamp AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') FROM sessions s JOIN messages m ON m.session_id = s.id - WHERE ` + where + ` AND m.timestamp IS NOT NULL` + WHERE ` + strings.Join(preds, " AND ") rows, err := s.pg.QueryContext( ctx, query, pb.args..., @@ -372,6 +465,31 @@ func (s *Store) filteredSessionIDs( return ids, nil } +// filteredSessionIDsModel returns the sessions that have at least one +// model-scoped message matching the hour/dow filter. It runs the shared scope +// reducer (with the day/hour filter) instead of a direct m.model predicate, so +// an empty-model user turn paired with a selected-model assistant keeps its +// session when the user turn falls in the selected hour. +func (s *Store) filteredSessionIDsModel( + ctx context.Context, f db.AnalyticsFilter, +) (map[string]bool, error) { + sessionIDs, err := s.analyticsModelCandidateSessionIDs(ctx, f) + if err != nil { + return nil, err + } + scope, err := s.resolveAnalyticsMessageScope(ctx, sessionIDs, f, false) + if err != nil { + return nil, err + } + ids := make(map[string]bool) + if scope != nil { + for id := range scope.MessagesBySession() { + ids[id] = true + } + } + return ids, nil +} + // bucketDate truncates a date to the start of its bucket. func bucketDate(date string, granularity string) string { t, err := time.Parse("2006-01-02", date) @@ -393,6 +511,170 @@ func bucketDate(date string, granularity string) string { } } +func (s *Store) getModelScopedToolCallCounts( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, +) (map[string]int, error) { + counts := make(map[string]int, len(sessionIDs)) + if len(sessionIDs) == 0 || strings.TrimSpace(f.Model) == "" { + return counts, nil + } + + flt := messageScopeFilter(f) + loc := analyticsLocation(f) + if err := pgQueryChunked(sessionIDs, func(chunk []string) error { + pb := ¶mBuilder{} + placeholders := pgInPlaceholders(chunk, pb) + rows, err := s.pg.QueryContext(ctx, ` + SELECT tc.session_id, m.model, + COALESCE(TO_CHAR(m.timestamp AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), ''), + COUNT(*) + FROM tool_calls tc + JOIN messages m + ON m.session_id = tc.session_id + AND m.ordinal = tc.message_ordinal + WHERE tc.session_id IN `+placeholders+` + GROUP BY tc.session_id, m.model, + COALESCE(TO_CHAR(m.timestamp AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), '')`, + pb.args..., + ) + if err != nil { + return fmt.Errorf( + "querying filtered analytics tool calls: %w", + err, + ) + } + defer rows.Close() + + for rows.Next() { + var sessionID, model, ts string + var count int + if err := rows.Scan(&sessionID, &model, &ts, &count); err != nil { + return fmt.Errorf( + "scanning filtered analytics tool calls: %w", + err, + ) + } + if _, ok := flt.Models[model]; !ok { + continue + } + parsed, has := localTime(ts, loc) + if !flt.MatchesDayHour(parsed, has) { + continue + } + counts[sessionID] += count + } + return rows.Err() + }); err != nil { + return nil, err + } + + return counts, nil +} + +func (s *Store) getAnalyticsActivityFilteredByModelTime( + ctx context.Context, + f db.AnalyticsFilter, + granularity string, +) (db.ActivityResponse, error) { + loc := analyticsLocation(f) + pb := ¶mBuilder{} + where := buildAnalyticsWhere(f, pgDateCol, pb) + var timeIDs map[string]bool + if f.HasTimeFilter() { + var err error + timeIDs, err = s.filteredSessionIDs(ctx, f) + if err != nil { + return db.ActivityResponse{}, err + } + } + + rows, err := s.pg.QueryContext(ctx, `SELECT id, `+pgDateCol+`, agent + FROM sessions + WHERE `+where, pb.args...) + if err != nil { + return db.ActivityResponse{}, + fmt.Errorf("querying analytics activity sessions: %w", err) + } + defer rows.Close() + + type sessionRow struct { + id, date, agent string + } + sessions := make([]sessionRow, 0) + sessionIDs := make([]string, 0) + for rows.Next() { + var id, agent string + var ts *time.Time + if err := rows.Scan(&id, &ts, &agent); err != nil { + return db.ActivityResponse{}, + fmt.Errorf("scanning analytics activity session: %w", err) + } + date := localDate(scanDateCol(ts), loc) + if !inDateRange(date, f.From, f.To) { + continue + } + if timeIDs != nil && !timeIDs[id] { + continue + } + sessions = append(sessions, sessionRow{id: id, date: date, agent: agent}) + sessionIDs = append(sessionIDs, id) + } + if err := rows.Err(); err != nil { + return db.ActivityResponse{}, + fmt.Errorf("iterating analytics activity sessions: %w", err) + } + + messageStats, err := s.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return db.ActivityResponse{}, err + } + toolCounts, err := s.getModelScopedToolCallCounts( + ctx, sessionIDs, f, + ) + if err != nil { + return db.ActivityResponse{}, err + } + + buckets := make(map[string]*db.ActivityEntry) + for _, session := range sessions { + bucket := bucketDate(session.date, granularity) + entry := buckets[bucket] + if entry == nil { + entry = &db.ActivityEntry{ + Date: bucket, + ByAgent: make(map[string]int), + } + buckets[bucket] = entry + } + entry.Sessions++ + stat := messageStats[session.id] + entry.Messages += stat.Messages + entry.UserMessages += stat.UserMessages + entry.AssistantMessages += stat.AssistantMessages + entry.ThinkingMessages += stat.ThinkingMessages + entry.ToolCalls += toolCounts[session.id] + entry.ByAgent[session.agent] += stat.Messages + } + + series := make([]db.ActivityEntry, 0, len(buckets)) + for _, entry := range buckets { + series = append(series, *entry) + } + sort.Slice(series, func(i, j int) bool { + return series[i].Date < series[j].Date + }) + return db.ActivityResponse{ + Granularity: granularity, + Series: series, + }, nil +} + // scanDateCol scans a TIMESTAMPTZ column and returns it as // an ISO-8601 string for client-side date processing. func scanDateCol(t *time.Time) string { @@ -402,6 +684,137 @@ func scanDateCol(t *time.Time) string { return FormatISO8601(*t) } +func (s *Store) getAnalyticsModelsForSessionIDs( + ctx context.Context, + sessionIDs []string, +) ([]string, error) { + if len(sessionIDs) == 0 { + return []string{}, nil + } + seen := make(map[string]struct{}, len(sessionIDs)) + unique := make([]string, 0, len(sessionIDs)) + for _, sessionID := range sessionIDs { + if _, ok := seen[sessionID]; ok { + continue + } + seen[sessionID] = struct{}{} + unique = append(unique, sessionID) + } + + modelSet := make(map[string]struct{}) + models := make([]string, 0) + if err := pgQueryChunked(unique, func(chunk []string) error { + pb := ¶mBuilder{} + placeholders := pgInPlaceholders(chunk, pb) + rows, err := s.pg.QueryContext(ctx, ` + SELECT DISTINCT model + FROM messages + WHERE session_id IN `+placeholders+` + AND COALESCE(model, '') <> '' + ORDER BY model`, + pb.args..., + ) + if err != nil { + return fmt.Errorf("querying analytics models: %w", err) + } + defer rows.Close() + for rows.Next() { + var model string + if err := rows.Scan(&model); err != nil { + return fmt.Errorf("scanning analytics model: %w", err) + } + if _, ok := modelSet[model]; ok { + continue + } + modelSet[model] = struct{}{} + models = append(models, model) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating analytics models: %w", err) + } + return nil + }); err != nil { + return nil, err + } + sort.Strings(models) + return models, nil +} + +func (s *Store) getAnalyticsModelsForSessionIDsFiltered( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, +) ([]string, error) { + if len(sessionIDs) == 0 { + return []string{}, nil + } + seen := make(map[string]struct{}, len(sessionIDs)) + unique := make([]string, 0, len(sessionIDs)) + for _, sessionID := range sessionIDs { + if _, ok := seen[sessionID]; ok { + continue + } + seen[sessionID] = struct{}{} + unique = append(unique, sessionID) + } + + filterModels := csvFilterValues(f.Model) + allowedModels := make(map[string]struct{}, len(filterModels)) + for _, model := range filterModels { + allowedModels[model] = struct{}{} + } + loc := analyticsLocation(f) + modelSet := make(map[string]struct{}) + models := make([]string, 0) + if err := pgQueryChunked(unique, func(chunk []string) error { + pb := ¶mBuilder{} + placeholders := pgInPlaceholders(chunk, pb) + rows, err := s.pg.QueryContext(ctx, ` + SELECT model, + COALESCE(TO_CHAR(timestamp AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), '') + FROM messages + WHERE session_id IN `+placeholders+` + AND COALESCE(model, '') <> ''`, + pb.args..., + ) + if err != nil { + return fmt.Errorf("querying filtered analytics models: %w", err) + } + defer rows.Close() + for rows.Next() { + var model, ts string + if err := rows.Scan(&model, &ts); err != nil { + return fmt.Errorf("scanning filtered analytics model: %w", err) + } + if len(allowedModels) > 0 { + if _, ok := allowedModels[model]; !ok { + continue + } + } + if f.HasTimeFilter() { + t, ok := localTime(ts, loc) + if !ok || !matchesTimeFilter(f, t) { + continue + } + } + if _, ok := modelSet[model]; ok { + continue + } + modelSet[model] = struct{}{} + models = append(models, model) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating filtered analytics models: %w", err) + } + return nil + }); err != nil { + return nil, err + } + sort.Strings(models) + return models, nil +} + // --- Summary --- // GetAnalyticsSummary returns aggregate statistics. @@ -441,6 +854,7 @@ func (s *Store) GetAnalyticsSummary( defer rows.Close() type sessionRow struct { + id string date string messages int agent string @@ -474,6 +888,7 @@ func (s *Store) GetAnalyticsSummary( continue } all = append(all, sessionRow{ + id: id, date: date, messages: mc, agent: agent, @@ -491,14 +906,35 @@ func (s *Store) GetAnalyticsSummary( var summary db.AnalyticsSummary summary.Agents = make(map[string]*db.AgentSummary) + summary.Models = []string{} if len(all) == 0 { return summary, nil } + if f.Model != "" { + sessionIDs := make([]string, 0, len(all)) + for _, r := range all { + sessionIDs = append(sessionIDs, r.id) + } + stats, err := s.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return db.AnalyticsSummary{}, err + } + for i := range all { + stat := stats[all[i].id] + all[i].messages = stat.Messages + all[i].outputTokens = stat.OutputTokens + all[i].hasTokens = stat.HasOutputTokens + } + } + days := make(map[string]bool) projects := make(map[string]int) msgCounts := make([]int, 0, len(all)) + sessionIDs := make([]string, 0, len(all)) for _, r := range all { summary.TotalSessions++ @@ -510,6 +946,7 @@ func (s *Store) GetAnalyticsSummary( days[r.date] = true projects[r.project] += r.messages msgCounts = append(msgCounts, r.messages) + sessionIDs = append(sessionIDs, r.id) if summary.Agents[r.agent] == nil { summary.Agents[r.agent] = &db.AgentSummary{} @@ -518,6 +955,19 @@ func (s *Store) GetAnalyticsSummary( summary.Agents[r.agent].Messages += r.messages } + var models []string + if strings.TrimSpace(f.Model) != "" || f.HasTimeFilter() { + models, err = s.getAnalyticsModelsForSessionIDsFiltered( + ctx, sessionIDs, f, + ) + } else { + models, err = s.getAnalyticsModelsForSessionIDs(ctx, sessionIDs) + } + if err != nil { + return db.AnalyticsSummary{}, err + } + summary.Models = models + summary.ActiveProjects = len(projects) summary.ActiveDays = len(days) summary.AvgMessages = math.Round( @@ -579,9 +1029,19 @@ func (s *Store) GetAnalyticsActivity( if granularity == "" { granularity = "day" } + if strings.TrimSpace(f.Model) != "" { + return s.getAnalyticsActivityFilteredByModelTime( + ctx, f, granularity, + ) + } loc := analyticsLocation(f) pb := ¶mBuilder{} - where := buildAnalyticsWhere(f, pgDateColS, pb) + where := buildAnalyticsWhereWithDate( + f, pgDateColS, pb, true, "s.id", + ) + preds := appendPGAnalyticsCSVFilter( + []string{where}, "m.model", f.Model, pb, + ) var timeIDs map[string]bool if f.HasTimeFilter() { @@ -596,7 +1056,7 @@ func (s *Store) GetAnalyticsActivity( m.role, m.has_thinking, COUNT(*) FROM sessions s LEFT JOIN messages m ON m.session_id = s.id - WHERE ` + where + ` + WHERE ` + strings.Join(preds, " AND ") + ` GROUP BY s.id, ` + pgDateColS + `, s.agent, m.role, m.has_thinking` @@ -680,7 +1140,7 @@ func (s *Store) GetAnalyticsActivity( err = pgQueryChunked(sessionIDs, func(chunk []string) error { return s.mergeActivityToolCalls( - ctx, chunk, sessionSeen, buckets, + ctx, chunk, sessionSeen, buckets, f.Model, ) }) if err != nil { @@ -710,13 +1170,25 @@ func (s *Store) mergeActivityToolCalls( chunk []string, sessionBucket map[string]string, buckets map[string]*db.ActivityEntry, + model string, ) error { pb := ¶mBuilder{} ph := pgInPlaceholders(chunk, pb) - q := `SELECT session_id, COUNT(*) - FROM tool_calls - WHERE session_id IN ` + ph + ` - GROUP BY session_id` + preds := []string{"tc.session_id IN " + ph} + preds = appendPGAnalyticsCSVFilter( + preds, "m.model", model, pb, + ) + q := `SELECT tc.session_id, COUNT(*) + FROM tool_calls tc` + if model != "" { + q += ` + JOIN messages m + ON m.session_id = tc.session_id + AND m.ordinal = tc.message_ordinal` + } + q += ` + WHERE ` + strings.Join(preds, " AND ") + ` + GROUP BY tc.session_id` rows, err := s.pg.QueryContext(ctx, q, pb.args...) if err != nil { return fmt.Errorf( @@ -868,6 +1340,15 @@ func (s *Store) GetAnalyticsHeatmap( } defer rows.Close() + type heatmapRow struct { + id string + date string + messages int + outputTokens int + hasTokens bool + } + + var heatmapRows []heatmapRow dayCounts := make(map[string]int) daySessions := make(map[string]int) dayOutputTokens := make(map[string]int) @@ -892,11 +1373,13 @@ func (s *Store) GetAnalyticsHeatmap( if timeIDs != nil && !timeIDs[id] { continue } - dayCounts[date] += mc - daySessions[date]++ - if hasTokens { - dayOutputTokens[date] += outputTokens - } + heatmapRows = append(heatmapRows, heatmapRow{ + id: id, + date: date, + messages: mc, + outputTokens: outputTokens, + hasTokens: hasTokens, + }) } if err := rows.Err(); err != nil { return db.HeatmapResponse{}, @@ -905,6 +1388,31 @@ func (s *Store) GetAnalyticsHeatmap( ) } + if f.Model != "" && (metric == "messages" || metric == "output_tokens") { + ids := make([]string, 0, len(heatmapRows)) + for _, row := range heatmapRows { + ids = append(ids, row.id) + } + stats, err := s.getAnalyticsFilteredMessageStats(ctx, ids, f) + if err != nil { + return db.HeatmapResponse{}, err + } + for i := range heatmapRows { + stat := stats[heatmapRows[i].id] + heatmapRows[i].messages = stat.Messages + heatmapRows[i].outputTokens = stat.OutputTokens + heatmapRows[i].hasTokens = stat.HasOutputTokens + } + } + + for _, row := range heatmapRows { + dayCounts[row.date] += row.messages + daySessions[row.date]++ + if row.hasTokens { + dayOutputTokens[row.date] += row.outputTokens + } + } + source := dayCounts switch metric { case "sessions": @@ -997,6 +1505,14 @@ func (s *Store) GetAnalyticsProjects( projectMap := make(map[string]*projectData) var projectOrder []string + type projectRow struct { + id string + project string + date string + messages int + agent string + } + var projectRows []projectRow for rows.Next() { var id, project, agent string @@ -1018,38 +1534,62 @@ func (s *Store) GetAnalyticsProjects( continue } - pd, ok := projectMap[project] + projectRows = append(projectRows, projectRow{ + id: id, + project: project, + date: date, + messages: mc, + agent: agent, + }) + } + if err := rows.Err(); err != nil { + return db.ProjectsAnalyticsResponse{}, + fmt.Errorf( + "iterating project rows: %w", err, + ) + } + + if f.Model != "" { + ids := make([]string, 0, len(projectRows)) + for _, row := range projectRows { + ids = append(ids, row.id) + } + counts, err := s.getAnalyticsFilteredMessageCounts( + ctx, ids, f, + ) + if err != nil { + return db.ProjectsAnalyticsResponse{}, err + } + for i := range projectRows { + projectRows[i].messages = counts[projectRows[i].id] + } + } + + for _, row := range projectRows { + pd, ok := projectMap[row.project] if !ok { pd = &projectData{ - name: project, + name: row.project, agents: make(map[string]int), days: make(map[string]int), } - projectMap[project] = pd - projectOrder = append( - projectOrder, project, - ) + projectMap[row.project] = pd + projectOrder = append(projectOrder, row.project) } pd.sessions++ - pd.messages += mc - pd.counts = append(pd.counts, mc) - pd.agents[agent]++ - pd.days[date] += mc + pd.messages += row.messages + pd.counts = append(pd.counts, row.messages) + pd.agents[row.agent]++ + pd.days[row.date] += row.messages - if pd.first == "" || date < pd.first { - pd.first = date + if pd.first == "" || row.date < pd.first { + pd.first = row.date } - if date > pd.last { - pd.last = date + if row.date > pd.last { + pd.last = row.date } } - if err := rows.Err(); err != nil { - return db.ProjectsAnalyticsResponse{}, - fmt.Errorf( - "iterating project rows: %w", err, - ) - } projects := make( []db.ProjectAnalytics, 0, len(projectMap), @@ -1106,16 +1646,22 @@ func (s *Store) GetAnalyticsProjects( func (s *Store) GetAnalyticsHourOfWeek( ctx context.Context, f db.AnalyticsFilter, ) (db.HourOfWeekResponse, error) { + if strings.TrimSpace(f.Model) != "" { + return s.getAnalyticsHourOfWeekFilteredByModel(ctx, f) + } loc := analyticsLocation(f) pb := ¶mBuilder{} - where := buildAnalyticsWhere(f, pgDateColS, pb) + where := buildAnalyticsWhereWithDate( + f, pgDateColS, pb, true, "s.id", + ) + preds := []string{where, "m.timestamp IS NOT NULL"} query := `SELECT ` + pgDateColS + `, TO_CHAR(m.timestamp AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') FROM sessions s JOIN messages m ON m.session_id = s.id - WHERE ` + where + ` AND m.timestamp IS NOT NULL` + WHERE ` + strings.Join(preds, " AND ") rows, err := s.pg.QueryContext( ctx, query, pb.args..., @@ -1158,18 +1704,88 @@ func (s *Store) GetAnalyticsHourOfWeek( ) } - cells := make([]db.HourOfWeekCell, 0, 168) - for d := range 7 { - for h := range 24 { - cells = append(cells, db.HourOfWeekCell{ - DayOfWeek: d, - Hour: h, - Messages: grid[d][h], - }) + return db.HourOfWeekResponseFromGrid(grid), nil +} + +// getAnalyticsHourOfWeekFilteredByModel buckets model-scoped messages by +// day-of-week and hour. It pairs empty-model user turns with their +// selected-model assistant via the shared scope reducer, so those turns appear +// in the heatmap consistently with the summary, activity, velocity, and trends +// panels. The heatmap is the control that sets the day/hour filter, so it +// clears DayOfWeek/Hour before scoping to keep showing the full grid, matching +// the no-model path. +// analyticsModelCandidateSessionIDs returns the date-filtered, model-scoped +// session IDs that feed the shared message-scope reducer. The day/hour filter +// is intentionally not applied here: callers that need it let the reducer apply +// it (so paired empty-model user turns are kept), while the hour-of-week +// heatmap clears it to show the full grid. +func (s *Store) analyticsModelCandidateSessionIDs( + ctx context.Context, f db.AnalyticsFilter, +) ([]string, error) { + loc := analyticsLocation(f) + pb := ¶mBuilder{} + where := buildAnalyticsWhereWithDate( + f, pgDateCol, pb, true, "id", + ) + + rows, err := s.pg.QueryContext(ctx, `SELECT id, `+pgDateCol+` + FROM sessions + WHERE `+where, pb.args...) + if err != nil { + return nil, fmt.Errorf("querying model candidate sessions: %w", err) + } + defer rows.Close() + + ids := make([]string, 0) + for rows.Next() { + var id string + var ts *time.Time + if err := rows.Scan(&id, &ts); err != nil { + return nil, fmt.Errorf("scanning model candidate session: %w", err) + } + if !inDateRange(localDate(scanDateCol(ts), loc), f.From, f.To) { + continue + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating model candidate sessions: %w", err) + } + return ids, nil +} + +func (s *Store) getAnalyticsHourOfWeekFilteredByModel( + ctx context.Context, f db.AnalyticsFilter, +) (db.HourOfWeekResponse, error) { + sessionIDs, err := s.analyticsModelCandidateSessionIDs(ctx, f) + if err != nil { + return db.HourOfWeekResponse{}, err + } + + scopeFilter := f + scopeFilter.DayOfWeek = nil + scopeFilter.Hour = nil + scope, err := s.resolveAnalyticsMessageScope( + ctx, sessionIDs, scopeFilter, false, + ) + if err != nil { + return db.HourOfWeekResponse{}, err + } + + var grid [7][24]int + if scope != nil { + for _, msgs := range scope.MessagesBySession() { + for _, m := range msgs { + if !m.HasLocalTime { + continue + } + dow := (int(m.LocalTime.Weekday()) + 6) % 7 + grid[dow][m.LocalTime.Hour()]++ + } } } - return db.HourOfWeekResponse{Cells: cells}, nil + return db.HourOfWeekResponseFromGrid(grid), nil } // --- Session Shape --- @@ -1278,6 +1894,7 @@ func (s *Store) GetAnalyticsSessionShape( ctx context.Context, f db.AnalyticsFilter, ) (db.SessionShapeResponse, error) { loc := analyticsLocation(f) + modelFilter := strings.TrimSpace(f.Model) != "" pb := ¶mBuilder{} where := buildAnalyticsWhere(f, pgDateCol, pb) @@ -1334,7 +1951,9 @@ func (s *Store) GetAnalyticsSessionShape( } totalCount++ - lengthCounts[lengthBucket(mc)]++ + if !modelFilter { + lengthCounts[lengthBucket(mc)]++ + } sessionIDs = append(sessionIDs, id) if durationSec != nil && *durationSec >= 0 { @@ -1351,7 +1970,29 @@ func (s *Store) GetAnalyticsSessionShape( } autonomyCounts := make(map[string]int) - if len(sessionIDs) > 0 { + if modelFilter && len(sessionIDs) > 0 { + stats, err := s.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return db.SessionShapeResponse{}, err + } + lengthCounts = make(map[string]int) + seen := make(map[string]struct{}, len(sessionIDs)) + for _, sessionID := range sessionIDs { + if _, ok := seen[sessionID]; ok { + continue + } + seen[sessionID] = struct{}{} + stat := stats[sessionID] + lengthCounts[lengthBucket(stat.Messages)]++ + if stat.UserMessages > 0 { + ratio := float64(stat.ToolUseMessages) / + float64(stat.UserMessages) + autonomyCounts[autonomyBucket(ratio)]++ + } + } + } else if len(sessionIDs) > 0 { err := pgQueryChunked(sessionIDs, func(chunk []string) error { return s.queryAutonomyChunk( @@ -1426,18 +2067,8 @@ func (s *Store) queryAutonomyChunk( func (s *Store) GetAnalyticsTools( ctx context.Context, f db.AnalyticsFilter, ) (db.ToolsAnalyticsResponse, error) { - loc := analyticsLocation(f) pb := ¶mBuilder{} - where := buildAnalyticsWhere(f, pgDateCol, pb) - - var timeIDs map[string]bool - if f.HasTimeFilter() { - var err error - timeIDs, err = s.filteredSessionIDs(ctx, f) - if err != nil { - return db.ToolsAnalyticsResponse{}, err - } - } + where := buildAnalyticsWhereWithoutDate(f, pb) sessQ := `SELECT id, ` + pgDateCol + `, agent FROM sessions WHERE ` + where @@ -1454,7 +2085,7 @@ func (s *Store) GetAnalyticsTools( defer sessRows.Close() type sessInfo struct { - date string + ts string agent string } sessionMap := make(map[string]sessInfo) @@ -1471,15 +2102,8 @@ func (s *Store) GetAnalyticsTools( "scanning tool session: %w", err, ) } - date := localDate(scanDateCol(ts), loc) - if !inDateRange(date, f.From, f.To) { - continue - } - if timeIDs != nil && !timeIDs[id] { - continue - } sessionMap[id] = sessInfo{ - date: date, agent: agent, + ts: scanDateCol(ts), agent: agent, } sessionIDs = append(sessionIDs, id) } @@ -1504,6 +2128,7 @@ func (s *Store) GetAnalyticsTools( sessionID string category string count int + date string } var toolRows []toolRow @@ -1511,10 +2136,21 @@ func (s *Store) GetAnalyticsTools( func(chunk []string) error { chunkPB := ¶mBuilder{} ph := pgInPlaceholders(chunk, chunkPB) - q := `SELECT session_id, category, COUNT(*) - FROM tool_calls - WHERE session_id IN ` + ph + ` - GROUP BY session_id, category` + preds := []string{"tc.session_id IN " + ph} + preds = appendPGAnalyticsCSVFilter( + preds, "m.model", f.Model, chunkPB, + ) + msgTSExpr := `COALESCE(TO_CHAR(m.timestamp AT TIME ZONE 'UTC', ` + + `'YYYY-MM-DD"T"HH24:MI:SS"Z"'), '')` + q := `SELECT tc.session_id, tc.category, COUNT(*), + ` + msgTSExpr + ` + FROM tool_calls tc + LEFT JOIN messages m + ON m.session_id = tc.session_id + AND m.ordinal = tc.message_ordinal` + q += ` + WHERE ` + strings.Join(preds, " AND ") + ` + GROUP BY tc.session_id, tc.category, ` + msgTSExpr rows, qErr := s.pg.QueryContext( ctx, q, chunkPB.args..., ) @@ -1525,19 +2161,30 @@ func (s *Store) GetAnalyticsTools( } defer rows.Close() for rows.Next() { - var sid, cat string + var sid, cat, ts string var count int if err := rows.Scan( - &sid, &cat, &count, + &sid, &cat, &count, &ts, ); err != nil { return fmt.Errorf( "scanning tool_call: %w", err, ) } + info, ok := sessionMap[sid] + if !ok { + continue + } + _, date, keep := f.ResolveSkillRowTime( + ts, info.ts, + ) + if !keep { + continue + } toolRows = append(toolRows, toolRow{ sessionID: sid, category: cat, count: count, + date: date, }) } return rows.Err() @@ -1563,7 +2210,7 @@ func (s *Store) GetAnalyticsTools( } agentCats[info.agent][tr.category] += tr.count - week := bucketDate(info.date, "week") + week := bucketDate(tr.date, "week") if trendBuckets[week] == nil { trendBuckets[week] = make(map[string]int) } @@ -1706,6 +2353,13 @@ func (s *Store) GetAnalyticsSkills( func(chunk []string) error { chunkPB := ¶mBuilder{} ph := pgInPlaceholders(chunk, chunkPB) + preds := []string{ + "tc.session_id IN " + ph, + "TRIM(COALESCE(tc.skill_name, '')) != ''", + } + preds = appendPGAnalyticsCSVFilter( + preds, "m.model", f.Model, chunkPB, + ) q := `SELECT tc.session_id, TRIM(COALESCE(tc.skill_name, '')), COUNT(*), @@ -1714,8 +2368,7 @@ func (s *Store) GetAnalyticsSkills( LEFT JOIN messages m ON m.session_id = tc.session_id AND m.ordinal = tc.message_ordinal - WHERE tc.session_id IN ` + ph + ` - AND TRIM(COALESCE(tc.skill_name, '')) != '' + WHERE ` + strings.Join(preds, " AND ") + ` GROUP BY tc.session_id, TRIM(COALESCE(tc.skill_name, '')), m.timestamp` @@ -1828,6 +2481,42 @@ func (s *Store) queryVelocityMsgs( return rows.Err() } +func (s *Store) getAnalyticsVelocityMessages( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, +) (map[string][]velocityMsg, error) { + sessionMsgs := make(map[string][]velocityMsg, len(sessionIDs)) + if len(sessionIDs) == 0 { + return sessionMsgs, nil + } + + loc := analyticsLocation(f) + if strings.TrimSpace(f.Model) == "" { + err := pgQueryChunked(sessionIDs, func(chunk []string) error { + return s.queryVelocityMsgs(ctx, chunk, loc, sessionMsgs) + }) + return sessionMsgs, err + } + + scope, err := s.resolveAnalyticsMessageScope(ctx, sessionIDs, f, false) + if err != nil { + return nil, err + } + if scope != nil { + for sid, timings := range scope.TimingBySession() { + for _, t := range timings { + sessionMsgs[sid] = append(sessionMsgs[sid], velocityMsg{ + role: t.Role, ts: t.Time, valid: t.Valid, + contentLength: t.ContentLength, + }) + } + } + } + + return sessionMsgs, nil +} + // complexityBucket returns the complexity label. func complexityBucket(mc int) string { switch { @@ -1960,54 +2649,73 @@ func (s *Store) GetAnalyticsVelocity( ByComplexity: []db.VelocityBreakdown{}, }, nil } + if strings.TrimSpace(f.Model) != "" { + stats, err := s.getAnalyticsFilteredMessageStats( + ctx, sessionIDs, f, + ) + if err != nil { + return db.VelocityResponse{}, err + } + for _, sid := range sessionIDs { + info := sessionMap[sid] + info.mc = stats[sid].Messages + sessionMap[sid] = info + } + } - sessionMsgs := make(map[string][]velocityMsg) - err = pgQueryChunked(sessionIDs, - func(chunk []string) error { - return s.queryVelocityMsgs( - ctx, chunk, loc, sessionMsgs, - ) - }) + sessionMsgs, err := s.getAnalyticsVelocityMessages( + ctx, sessionIDs, f, + ) if err != nil { return db.VelocityResponse{}, err } - toolCountMap := make(map[string]int) - err = pgQueryChunked(sessionIDs, - func(chunk []string) error { - chunkPB := ¶mBuilder{} - ph := pgInPlaceholders(chunk, chunkPB) - q := `SELECT session_id, COUNT(*) - FROM tool_calls - WHERE session_id IN ` + ph + ` - GROUP BY session_id` - rows, qErr := s.pg.QueryContext( - ctx, q, chunkPB.args..., - ) - if qErr != nil { - return fmt.Errorf( - "querying velocity tool_calls: %w", - qErr, + var toolCountMap map[string]int + if strings.TrimSpace(f.Model) != "" { + toolCountMap, err = s.getModelScopedToolCallCounts( + ctx, sessionIDs, f, + ) + if err != nil { + return db.VelocityResponse{}, err + } + } else { + toolCountMap = make(map[string]int) + err = pgQueryChunked(sessionIDs, + func(chunk []string) error { + chunkPB := ¶mBuilder{} + ph := pgInPlaceholders(chunk, chunkPB) + q := `SELECT session_id, COUNT(*) + FROM tool_calls + WHERE session_id IN ` + ph + ` + GROUP BY session_id` + rows, qErr := s.pg.QueryContext( + ctx, q, chunkPB.args..., ) - } - defer rows.Close() - for rows.Next() { - var sid string - var count int - if err := rows.Scan( - &sid, &count, - ); err != nil { + if qErr != nil { return fmt.Errorf( - "scanning velocity tool_call: %w", - err, + "querying velocity tool_calls: %w", + qErr, ) } - toolCountMap[sid] = count - } - return rows.Err() - }) - if err != nil { - return db.VelocityResponse{}, err + defer rows.Close() + for rows.Next() { + var sid string + var count int + if err := rows.Scan( + &sid, &count, + ); err != nil { + return fmt.Errorf( + "scanning velocity tool_call: %w", + err, + ) + } + toolCountMap[sid] = count + } + return rows.Err() + }) + if err != nil { + return db.VelocityResponse{}, err + } } overall := &velocityAccumulator{} @@ -2207,10 +2915,14 @@ func (s *Store) GetAnalyticsTopSessions( } needsGoSort := metric == "duration" + needsFilteredMessageSort := strings.TrimSpace(f.Model) != "" && + (metric == "messages" || metric == "output_tokens") orderExpr := "message_count DESC, id ASC" switch metric { case "output_tokens": - where += " AND has_total_output_tokens = TRUE" + if strings.TrimSpace(f.Model) == "" { + where += " AND has_total_output_tokens = TRUE" + } orderExpr = "total_output_tokens DESC, id ASC" case "duration": where += " AND started_at IS NOT NULL" + @@ -2221,7 +2933,7 @@ func (s *Store) GetAnalyticsTopSessions( } limitClause := " LIMIT 1000" - if f.HasTimeFilter() || needsGoSort { + if f.HasTimeFilter() || needsGoSort || needsFilteredMessageSort { limitClause = "" } activeDurationSelectExpr := fmt.Sprintf(`COALESCE( @@ -2324,6 +3036,40 @@ func (s *Store) GetAnalyticsTopSessions( ) } + if needsFilteredMessageSort { + sessionIDs := make([]string, 0, len(sessions)) + for _, session := range sessions { + sessionIDs = append(sessionIDs, session.ID) + } + stats, err := s.getAnalyticsFilteredMessageStats(ctx, sessionIDs, f) + if err != nil { + return db.TopSessionsResponse{}, err + } + filtered := sessions[:0] + for i := range sessions { + stat := stats[sessions[i].ID] + sessions[i].MessageCount = stat.Messages + sessions[i].OutputTokens = stat.OutputTokens + if metric == "output_tokens" && !stat.HasOutputTokens { + continue + } + filtered = append(filtered, sessions[i]) + } + sessions = filtered + sort.SliceStable(sessions, func(i, j int) bool { + if metric == "output_tokens" { + if sessions[i].OutputTokens != sessions[j].OutputTokens { + return sessions[i].OutputTokens > + sessions[j].OutputTokens + } + } else if sessions[i].MessageCount != sessions[j].MessageCount { + return sessions[i].MessageCount > + sessions[j].MessageCount + } + return sessions[i].ID < sessions[j].ID + }) + } + sessions = rankTopSessions(sessions, needsGoSort) return db.TopSessionsResponse{ @@ -2336,7 +3082,10 @@ func (s *Store) GetAnalyticsTopSessions( // Mirrors the SQLite implementation: select per-session signal // columns, apply analytics filters, then hand the rows to the // shared db.AggregateSignals so the response shape stays -// identical across stores. +// identical across stores. Signals stay session-scoped under a +// model filter (totals are session-level aggregates over sessions +// that used the model, not re-attributed per model); see the +// SQLite GetAnalyticsSignals for the rationale. func (s *Store) GetAnalyticsSignals( ctx context.Context, f db.AnalyticsFilter, ) (db.SignalsAnalyticsResponse, error) { @@ -2444,7 +3193,7 @@ func (s *Store) GetAnalyticsSignalSessions( return db.SignalSessionsResponse{}, err } candidates := db.SignalCandidates(rows, signal, limit) - messages, err := s.signalMessages(ctx, candidates) + messages, err := s.signalMessages(ctx, candidates, f) if err != nil { return db.SignalSessionsResponse{}, err } @@ -2547,6 +3296,7 @@ func scanPGSignalRow( func (s *Store) signalMessages( ctx context.Context, rows []db.SignalRow, + f db.AnalyticsFilter, ) (map[string][]db.SignalMessage, error) { out := make(map[string][]db.SignalMessage, len(rows)) if len(rows) == 0 { @@ -2556,6 +3306,27 @@ func (s *Store) signalMessages( for _, r := range rows { ids = append(ids, r.ID) } + if strings.TrimSpace(f.Model) != "" { + rowsBySession, err := s.getAnalyticsModelScopedMessages(ctx, ids, f) + if err != nil { + return nil, err + } + for sessionID, scopedRows := range rowsBySession { + for _, row := range scopedRows { + out[sessionID] = append(out[sessionID], db.SignalMessage{ + SessionID: row.SessionID, + Ordinal: row.Ordinal, + Role: row.Role, + Content: row.Content, + Timestamp: row.Timestamp, + IsSystem: row.IsSystem, + HasToolUse: row.HasToolUse, + }) + } + } + return out, nil + } + filterModels := csvFilterValues(f.Model) err := pgQueryChunked(ids, func(chunk []string) error { pb := ¶mBuilder{} placeholders := make([]string, 0, len(chunk)) @@ -2567,7 +3338,20 @@ func (s *Store) signalMessages( 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'), ''), is_system, has_tool_use FROM messages - WHERE session_id IN (` + strings.Join(placeholders, ",") + `) + WHERE session_id IN (` + strings.Join(placeholders, ",") + `)` + if len(filterModels) == 1 { + q += ` AND model = ` + pb.add(filterModels[0]) + } else if len(filterModels) > 1 { + modelPlaceholders := make([]string, 0, len(filterModels)) + for _, model := range filterModels { + modelPlaceholders = append( + modelPlaceholders, + pb.add(model), + ) + } + q += ` AND model IN (` + strings.Join(modelPlaceholders, ",") + `)` + } + q += ` ORDER BY session_id, ordinal` msgRows, err := s.pg.QueryContext(ctx, q, pb.args...) if err != nil { diff --git a/internal/postgres/analytics_model_pgtest_test.go b/internal/postgres/analytics_model_pgtest_test.go new file mode 100644 index 000000000..afcd8e62f --- /dev/null +++ b/internal/postgres/analytics_model_pgtest_test.go @@ -0,0 +1,1344 @@ +//go:build pgtest + +package postgres + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +func TestStoreGetAnalyticsActivityModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_activity_model_messages_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'activity-model-messages-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:30:00Z'::timestamptz, + 2, 1 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('activity-model-messages-001', 0, 'user', 'q', + '2024-06-01T09:00:00Z'::timestamptz, 1, FALSE, 'gpt-4o'), + ('activity-model-messages-001', 1, 'assistant', 'a', + '2024-06-01T09:05:00Z'::timestamptz, 1, FALSE, 'claude-3-5-sonnet')`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsActivity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "day") + require.NoError(t, err, "GetAnalyticsActivity") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 1, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].UserMessages, "UserMessages") + assert.Equal(t, 0, resp.Series[0].AssistantMessages, + "AssistantMessages") +} + +func TestStoreGetAnalyticsActivityModelFilterKeepsNullTimestampSessionsWithoutTimeFilter( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_activity_model_null_ts_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'activity-model-null-ts-001', 'test-machine', 'alpha', 'gpt', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:30:00Z'::timestamptz, + 2, 1 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('activity-model-null-ts-001', 0, 'user', 'q', + '2024-06-01T09:00:00Z'::timestamptz, 1, FALSE, ''), + ('activity-model-null-ts-001', 1, 'assistant', 'a', + NULL, 1, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsActivity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "day") + require.NoError(t, err, "GetAnalyticsActivity") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 2, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].UserMessages, "UserMessages") + assert.Equal(t, 1, resp.Series[0].AssistantMessages, + "AssistantMessages") +} + +func TestStoreGetAnalyticsActivityModelAndHourFilterUseSameMessage( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_activity_model_hour_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'activity-model-hour-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:30:00Z'::timestamptz, + 2, 1 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('activity-model-hour-001', 0, 'user', 'q', + '2024-06-01T09:00:00Z'::timestamptz, 1, FALSE, 'gpt-4o'), + ('activity-model-hour-001', 1, 'assistant', 'a', + '2024-06-01T10:00:00Z'::timestamptz, 1, FALSE, 'claude-3-5-sonnet')`) + require.NoError(t, err, "insert messages") + + hour := 10 + resp, err := store.GetAnalyticsActivity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "day") + require.NoError(t, err, "GetAnalyticsActivity") + assert.Empty(t, resp.Series, "Series") +} + +func TestStoreGetAnalyticsActivityModelAndHourFilterKeepsPairedUserTurn( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_activity_model_hour_paired_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'activity-model-hour-paired-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:30:00Z'::timestamptz, + 2, 1 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('activity-model-hour-paired-001', 0, 'user', 'q', + '2024-06-01T09:00:00Z'::timestamptz, 1, FALSE, ''), + ('activity-model-hour-paired-001', 1, 'assistant', 'a', + '2024-06-01T10:00:00Z'::timestamptz, 1, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + + // The paired empty-model user turn sits in hour 9; the gpt-4o assistant in + // hour 10. Filtering by hour 9 must keep the session via the user turn. + hour := 9 + resp, err := store.GetAnalyticsActivity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "day") + require.NoError(t, err, "GetAnalyticsActivity") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 1, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].UserMessages, "UserMessages") + assert.Equal(t, 0, resp.Series[0].AssistantMessages, + "AssistantMessages") +} + +func TestStoreGetAnalyticsHeatmapSessionsModelAndHourFilterKeepsPairedUserTurn( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_heatmap_sessions_paired_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'heatmap-sessions-paired-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:30:00Z'::timestamptz, + 2, 1 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('heatmap-sessions-paired-001', 0, 'user', 'q', + '2024-06-01T09:00:00Z'::timestamptz, 1, FALSE, ''), + ('heatmap-sessions-paired-001', 1, 'assistant', 'a', + '2024-06-01T10:00:00Z'::timestamptz, 1, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + + // Empty-model user turn at hour 9 paired with a gpt-4o assistant at hour + // 10. The sessions heatmap must keep the session via the paired user turn. + hour := 9 + resp, err := store.GetAnalyticsHeatmap(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "sessions") + require.NoError(t, err, "GetAnalyticsHeatmap") + require.Len(t, resp.Entries, 1, "len(Entries)") + assert.Equal(t, 1, resp.Entries[0].Value, "Value") +} + +func TestStoreGetAnalyticsTopSessionsDurationModelAndHourFilterKeepsPairedUserTurn( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_top_duration_paired_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'top-duration-paired-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:00:00Z'::timestamptz, + 2, 1 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('top-duration-paired-001', 0, 'user', 'q', + '2024-06-01T09:00:00Z'::timestamptz, 1, FALSE, ''), + ('top-duration-paired-001', 1, 'assistant', 'a', + '2024-06-01T10:00:00Z'::timestamptz, 1, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + + // Empty-model user turn at hour 9 paired with a gpt-4o assistant at hour + // 10. Ranking by duration under the gpt-4o + hour-9 filter must keep the + // session via the paired user turn. + hour := 9 + resp, err := store.GetAnalyticsTopSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "duration") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 1, "len(Sessions)") + assert.Equal(t, "top-duration-paired-001", resp.Sessions[0].ID, "ID") +} + +func TestStoreGetAnalyticsActivityModelAndHourFilterCountsOnlyMatchingHourRows( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_activity_model_hour_rows_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'activity-model-hour-rows-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:30:00Z'::timestamptz, + 2, 0 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('activity-model-hour-rows-001', 0, 'assistant', 'read', + '2024-06-01T09:00:00Z'::timestamptz, 4, FALSE, 'gpt-4o'), + ('activity-model-hour-rows-001', 1, 'assistant', 'grep', + '2024-06-01T10:00:00Z'::timestamptz, 4, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO tool_calls ( + session_id, tool_name, category, call_index, + message_ordinal + ) VALUES + ('activity-model-hour-rows-001', 'Read', 'Read', 0, 0), + ('activity-model-hour-rows-001', 'Bash', 'Bash', 1, 0), + ('activity-model-hour-rows-001', 'Grep', 'Grep', 0, 1)`) + require.NoError(t, err, "insert tool_calls") + + hour := 10 + resp, err := store.GetAnalyticsActivity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "day") + require.NoError(t, err, "GetAnalyticsActivity") + require.Len(t, resp.Series, 1, "len(Series)") + assert.Equal(t, 1, resp.Series[0].Sessions, "Sessions") + assert.Equal(t, 1, resp.Series[0].Messages, "Messages") + assert.Equal(t, 1, resp.Series[0].AssistantMessages, + "AssistantMessages") + assert.Equal(t, 1, resp.Series[0].ToolCalls, "ToolCalls") +} + +func TestStoreGetAnalyticsHourOfWeekModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_hour_of_week_model_messages_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'hour-of-week-model-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:30:00Z'::timestamptz, + 2, 1 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('hour-of-week-model-001', 0, 'user', 'q', + '2024-06-01T09:00:00Z'::timestamptz, 1, FALSE, 'gpt-4o'), + ('hour-of-week-model-001', 1, 'assistant', 'a', + '2024-06-01T10:00:00Z'::timestamptz, 1, FALSE, 'claude-3-5-sonnet')`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsHourOfWeek(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsHourOfWeek") + assert.Equal(t, 1, findPGHOWCell(resp.Cells, 5, 9), "Sat 09:00") + assert.Equal(t, 0, findPGHOWCell(resp.Cells, 5, 10), "Sat 10:00") +} + +func TestStoreGetAnalyticsHourOfWeekModelFilterIncludesPairedUserTurns( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_hour_of_week_model_paired_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'hour-of-week-paired-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:30:00Z'::timestamptz, + 2, 1 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('hour-of-week-paired-001', 0, 'user', 'q', + '2024-06-01T09:00:00Z'::timestamptz, 1, FALSE, ''), + ('hour-of-week-paired-001', 1, 'assistant', 'a', + '2024-06-01T10:00:00Z'::timestamptz, 1, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsHourOfWeek(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsHourOfWeek") + assert.Equal(t, 1, findPGHOWCell(resp.Cells, 5, 9), + "paired empty-model user turn at Sat 09:00") + assert.Equal(t, 1, findPGHOWCell(resp.Cells, 5, 10), + "selected-model assistant at Sat 10:00") +} + +func TestStoreGetAnalyticsToolsModelFilterCountsOnlyMatchingToolCalls( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_tools_model_calls_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'tools-model-calls-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:30:00Z'::timestamptz, + 2, 0 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('tools-model-calls-001', 0, 'assistant', 'read', + '2024-06-01T09:00:00Z'::timestamptz, 4, FALSE, 'gpt-4o'), + ('tools-model-calls-001', 1, 'assistant', 'grep', + '2024-06-01T09:05:00Z'::timestamptz, 4, FALSE, 'claude-3-5-sonnet')`) + require.NoError(t, err, "insert messages") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO tool_calls ( + session_id, tool_name, category, call_index, + message_ordinal + ) VALUES + ('tools-model-calls-001', 'Read', 'Read', 0, 0), + ('tools-model-calls-001', 'Bash', 'Bash', 1, 0), + ('tools-model-calls-001', 'Grep', 'Grep', 0, 1)`) + require.NoError(t, err, "insert tool_calls") + + resp, err := store.GetAnalyticsTools(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsTools") + assert.Equal(t, 2, resp.TotalCalls, "TotalCalls") + require.Len(t, resp.ByCategory, 2, "len(ByCategory)") + + catMap := make(map[string]int) + for _, c := range resp.ByCategory { + catMap[c.Category] = c.Count + } + assert.Equal(t, 1, catMap["Read"], "Read") + assert.Equal(t, 1, catMap["Bash"], "Bash") + assert.Zero(t, catMap["Grep"], "Grep") +} + +func TestStoreGetAnalyticsToolsModelAndHourFilterCountsOnlyMatchingHourToolCalls( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_tools_model_hour_calls_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'tools-model-hour-calls-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:30:00Z'::timestamptz, + 2, 0 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('tools-model-hour-calls-001', 0, 'assistant', 'read', + '2024-06-01T09:00:00Z'::timestamptz, 4, FALSE, 'gpt-4o'), + ('tools-model-hour-calls-001', 1, 'assistant', 'grep', + '2024-06-01T10:00:00Z'::timestamptz, 4, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO tool_calls ( + session_id, tool_name, category, call_index, + message_ordinal + ) VALUES + ('tools-model-hour-calls-001', 'Read', 'Read', 0, 0), + ('tools-model-hour-calls-001', 'Bash', 'Bash', 1, 0), + ('tools-model-hour-calls-001', 'Grep', 'Grep', 0, 1)`) + require.NoError(t, err, "insert tool_calls") + + hour := 10 + resp, err := store.GetAnalyticsTools(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }) + require.NoError(t, err, "GetAnalyticsTools") + assert.Equal(t, 1, resp.TotalCalls, "TotalCalls") + require.Len(t, resp.ByCategory, 1, "len(ByCategory)") + assert.Equal(t, "Grep", resp.ByCategory[0].Category, "Category") + assert.Equal(t, 1, resp.ByCategory[0].Count, "Count") +} + +func TestStoreGetAnalyticsSkillsModelFilterCountsOnlyMatchingSkillCalls( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_skills_model_calls_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'skills-model-calls-001', 'test-machine', 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:30:00Z'::timestamptz, + 2, 0 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('skills-model-calls-001', 0, 'assistant', 'review', + '2024-06-01T09:00:00Z'::timestamptz, 6, FALSE, 'gpt-4o'), + ('skills-model-calls-001', 1, 'assistant', 'write', + '2024-06-01T09:05:00Z'::timestamptz, 5, FALSE, 'claude-3-5-sonnet')`) + require.NoError(t, err, "insert messages") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO tool_calls ( + session_id, tool_name, category, call_index, + skill_name, message_ordinal + ) VALUES + ('skills-model-calls-001', 'Skill', 'Skill', 0, 'review-code', 0), + ('skills-model-calls-001', 'Skill', 'Skill', 0, 'write-tests', 1)`) + require.NoError(t, err, "insert tool_calls") + + resp, err := store.GetAnalyticsSkills(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsSkills") + assert.Equal(t, 1, resp.TotalSkillCalls, "TotalSkillCalls") + assert.Equal(t, 1, resp.DistinctSkills, "DistinctSkills") + require.Len(t, resp.BySkill, 1, "len(BySkill)") + assert.Equal(t, "review-code", resp.BySkill[0].SkillName, "SkillName") + assert.Equal(t, 1, resp.BySkill[0].CallCount, "CallCount") + assert.Equal(t, "2024-06-01T09:00:00Z", resp.BySkill[0].LastUsedAt, + "LastUsedAt") +} + +func TestStoreGetAnalyticsSummaryModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_summary_model_messages_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'summary-model-messages-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:30:00Z'::timestamptz, + 2, 0 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('summary-model-messages-001', 0, 'assistant', 'gpt', + '2024-06-01T09:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o'), + ('summary-model-messages-001', 1, 'assistant', 'claude', + '2024-06-01T09:05:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet')`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsSummary(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsSummary") + assert.Equal(t, 1, resp.TotalSessions, "TotalSessions") + assert.Equal(t, 1, resp.TotalMessages, "TotalMessages") + assert.Equal(t, []string{"gpt-4o"}, resp.Models, "Models") + assert.Equal(t, 1.0, resp.AvgMessages, "AvgMessages") + assert.Equal(t, 1, resp.MedianMessages, "MedianMessages") + assert.Equal(t, 1, resp.P90Messages, "P90Messages") + require.Contains(t, resp.Agents, "mixed") + assert.Equal(t, 1, resp.Agents["mixed"].Messages, "AgentMessages") +} + +func TestStoreGetAnalyticsSummaryModelsUseMatchingHourRowsOnly( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_summary_hour_models_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'summary-hour-models-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:30:00Z'::timestamptz, + 2, 0 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('summary-hour-models-001', 0, 'assistant', 'gpt', + '2024-06-01T09:05:00Z'::timestamptz, 3, FALSE, 'gpt-4o'), + ('summary-hour-models-001', 1, 'assistant', 'claude', + '2024-06-01T10:05:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet')`) + require.NoError(t, err, "insert messages") + + hour := 9 + resp, err := store.GetAnalyticsSummary(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Hour: &hour, + }) + require.NoError(t, err, "GetAnalyticsSummary") + assert.Equal(t, 1, resp.TotalSessions, "TotalSessions") + assert.Equal(t, []string{"gpt-4o"}, resp.Models, "Models") +} + +func TestStoreGetAnalyticsSummaryModelFilterUsesFilteredOutputTokens( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_summary_model_output_tokens_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count, + total_output_tokens, has_total_output_tokens + ) VALUES + ( + 'summary-output-mixed-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T10:00:00Z'::timestamptz, + '2024-06-01T10:30:00Z'::timestamptz, + 2, 0, 111, TRUE + ), + ( + 'summary-output-uncovered-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T10:40:00Z'::timestamptz, + '2024-06-01T11:00:00Z'::timestamptz, + 2, 0, 90, TRUE + )`) + require.NoError(t, err, "insert sessions") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model, + output_tokens, has_output_tokens + ) VALUES + ('summary-output-mixed-001', 0, 'assistant', 'gpt', + '2024-06-01T10:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o', + 11, TRUE), + ('summary-output-mixed-001', 1, 'assistant', 'claude', + '2024-06-01T10:05:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet', + 100, TRUE), + ('summary-output-uncovered-001', 0, 'assistant', 'gpt', + '2024-06-01T10:40:00Z'::timestamptz, 3, FALSE, 'gpt-4o', + 0, FALSE), + ('summary-output-uncovered-001', 1, 'assistant', 'claude', + '2024-06-01T10:45:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet', + 90, TRUE)`) + require.NoError(t, err, "insert messages") + + hour := 10 + resp, err := store.GetAnalyticsSummary(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }) + require.NoError(t, err, "GetAnalyticsSummary") + assert.Equal(t, 2, resp.TotalSessions, "TotalSessions") + assert.Equal(t, 2, resp.TotalMessages, "TotalMessages") + assert.Equal(t, []string{"gpt-4o"}, resp.Models, "Models") + assert.Equal(t, 11, resp.TotalOutputTokens, "TotalOutputTokens") + assert.Equal(t, 1, resp.TokenReportingSessions, "TokenReportingSessions") +} + +func TestStoreGetAnalyticsProjectsModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_projects_model_messages_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'projects-model-messages-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:30:00Z'::timestamptz, + 2, 0 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('projects-model-messages-001', 0, 'assistant', 'gpt', + '2024-06-01T09:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o'), + ('projects-model-messages-001', 1, 'assistant', 'claude', + '2024-06-01T09:05:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet')`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsProjects(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsProjects") + require.Len(t, resp.Projects, 1, "len(Projects)") + assert.Equal(t, 1, resp.Projects[0].Messages, "Messages") + assert.Equal(t, 1.0, resp.Projects[0].AvgMessages, "AvgMessages") + assert.Equal(t, 1, resp.Projects[0].MedianMessages, "MedianMessages") + assert.Equal(t, 1.0, resp.Projects[0].DailyTrend, "DailyTrend") +} + +func TestStoreGetAnalyticsHeatmapModelFilterCountsOnlyMatchingMessages( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_heatmap_model_messages_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'heatmap-model-messages-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:30:00Z'::timestamptz, + 2, 0 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('heatmap-model-messages-001', 0, 'assistant', 'gpt', + '2024-06-01T09:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o'), + ('heatmap-model-messages-001', 1, 'assistant', 'claude', + '2024-06-01T09:05:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet')`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsHeatmap(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "messages") + require.NoError(t, err, "GetAnalyticsHeatmap") + require.Len(t, resp.Entries, 1, "len(Entries)") + assert.Equal(t, 1, resp.Entries[0].Value, "Value") +} + +func TestStoreGetAnalyticsHeatmapModelFilterUsesFilteredOutputTokens( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_heatmap_model_output_tokens_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count, + total_output_tokens, has_total_output_tokens + ) VALUES + ( + 'heatmap-output-mixed-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T10:00:00Z'::timestamptz, + '2024-06-01T10:30:00Z'::timestamptz, + 2, 0, 111, TRUE + ), + ( + 'heatmap-output-uncovered-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T10:40:00Z'::timestamptz, + '2024-06-01T11:00:00Z'::timestamptz, + 2, 0, 90, TRUE + )`) + require.NoError(t, err, "insert sessions") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model, + output_tokens, has_output_tokens + ) VALUES + ('heatmap-output-mixed-001', 0, 'assistant', 'gpt', + '2024-06-01T10:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o', + 11, TRUE), + ('heatmap-output-mixed-001', 1, 'assistant', 'claude', + '2024-06-01T10:05:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet', + 100, TRUE), + ('heatmap-output-uncovered-001', 0, 'assistant', 'gpt', + '2024-06-01T10:40:00Z'::timestamptz, 3, FALSE, 'gpt-4o', + 0, FALSE), + ('heatmap-output-uncovered-001', 1, 'assistant', 'claude', + '2024-06-01T10:45:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet', + 90, TRUE)`) + require.NoError(t, err, "insert messages") + + hour := 10 + resp, err := store.GetAnalyticsHeatmap(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", Hour: &hour, + }, "output_tokens") + require.NoError(t, err, "GetAnalyticsHeatmap") + require.Len(t, resp.Entries, 1, "len(Entries)") + assert.Equal(t, 11, resp.Entries[0].Value, "Value") +} + +func TestStoreGetAnalyticsTopSessionsMessagesUseFilteredModelCounts( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_top_sessions_model_messages_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES + ( + 'top-model-mixed-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:00:00Z'::timestamptz, + 3, 0 + ), + ( + 'top-model-gpt-001', 'test-machine', 'alpha', 'gpt', + '2024-06-01T11:00:00Z'::timestamptz, + '2024-06-01T12:00:00Z'::timestamptz, + 2, 0 + )`) + require.NoError(t, err, "insert sessions") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('top-model-mixed-001', 0, 'assistant', 'gpt', + '2024-06-01T09:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o'), + ('top-model-mixed-001', 1, 'assistant', 'claude', + '2024-06-01T09:05:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet'), + ('top-model-mixed-001', 2, 'assistant', 'claude', + '2024-06-01T09:06:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet'), + ('top-model-gpt-001', 0, 'assistant', 'gpt', + '2024-06-01T11:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o'), + ('top-model-gpt-001', 1, 'assistant', 'gpt', + '2024-06-01T11:05:00Z'::timestamptz, 3, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsTopSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "messages") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 2, "len(Sessions)") + assert.Equal(t, "top-model-gpt-001", resp.Sessions[0].ID, "top session") + assert.Equal(t, 2, resp.Sessions[0].MessageCount, "top MessageCount") + assert.Equal(t, "top-model-mixed-001", resp.Sessions[1].ID, "second session") + assert.Equal(t, 1, resp.Sessions[1].MessageCount, "second MessageCount") +} + +func TestStoreGetAnalyticsTopSessionsOutputTokensUseFilteredModelTotals( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_top_sessions_model_output_tokens_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count, + total_output_tokens, has_total_output_tokens + ) VALUES + ( + 'top-output-mixed-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:00:00Z'::timestamptz, + 2, 0, 510, TRUE + ), + ( + 'top-output-gpt-001', 'test-machine', 'alpha', 'gpt', + '2024-06-01T11:00:00Z'::timestamptz, + '2024-06-01T12:00:00Z'::timestamptz, + 1, 0, 30, TRUE + ), + ( + 'top-output-uncovered-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T13:00:00Z'::timestamptz, + '2024-06-01T14:00:00Z'::timestamptz, + 2, 0, 900, TRUE + )`) + require.NoError(t, err, "insert sessions") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model, + output_tokens, has_output_tokens + ) VALUES + ('top-output-mixed-001', 0, 'assistant', 'gpt', + '2024-06-01T09:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o', + 10, TRUE), + ('top-output-mixed-001', 1, 'assistant', 'claude', + '2024-06-01T09:05:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet', + 500, TRUE), + ('top-output-gpt-001', 0, 'assistant', 'gpt', + '2024-06-01T11:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o', + 30, TRUE), + ('top-output-uncovered-001', 0, 'assistant', 'gpt', + '2024-06-01T13:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o', + 0, FALSE), + ('top-output-uncovered-001', 1, 'assistant', 'claude', + '2024-06-01T13:05:00Z'::timestamptz, 6, FALSE, 'claude-3-5-sonnet', + 900, TRUE)`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsTopSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "output_tokens") + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 2, "len(Sessions)") + assert.Equal(t, "top-output-gpt-001", resp.Sessions[0].ID, "top session") + assert.Equal(t, 30, resp.Sessions[0].OutputTokens, + "top OutputTokens") + assert.Equal(t, "top-output-mixed-001", resp.Sessions[1].ID, + "second session") + assert.Equal(t, 10, resp.Sessions[1].OutputTokens, + "second OutputTokens") +} + +// The model-scoped messages/output_tokens path drops the SQL LIMIT to re-rank +// by filtered counts in Go, so it must reapply the top-10 cap before returning +// like SQLite and DuckDB. Seed more than 10 model-matching sessions and assert +// both metrics still cap at 10 with the highest-count session first. +func TestStoreGetAnalyticsTopSessionsModelFilterCapsAtTen( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_top_sessions_model_cap_test") + ctx := context.Background() + + const sessionCount = 12 + var sessionRows, messageRows []string + for k := 0; k < sessionCount; k++ { + id := fmt.Sprintf("top-cap-%02d", k) + // Distinct gpt-4o message counts 1..12 so ranking is unambiguous. + msgs := k + 1 + sessionRows = append(sessionRows, fmt.Sprintf( + `('%s', 'test-machine', 'alpha', 'gpt', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T10:00:00Z'::timestamptz, + %d, 0, %d, TRUE)`, id, msgs, msgs*10)) + for o := 0; o < msgs; o++ { + messageRows = append(messageRows, fmt.Sprintf( + `('%s', %d, 'assistant', 'gpt', + '2024-06-01T09:00:00Z'::timestamptz, 3, FALSE, 'gpt-4o', + 10, TRUE)`, id, o)) + } + } + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count, + total_output_tokens, has_total_output_tokens + ) VALUES `+strings.Join(sessionRows, ",")) + require.NoError(t, err, "insert sessions") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model, + output_tokens, has_output_tokens + ) VALUES `+strings.Join(messageRows, ",")) + require.NoError(t, err, "insert messages") + + for _, tc := range []struct { + metric string + topVal int + field func(db.TopSession) int + }{ + {"messages", 12, func(s db.TopSession) int { return s.MessageCount }}, + {"output_tokens", 120, func(s db.TopSession) int { return s.OutputTokens }}, + } { + t.Run(tc.metric, func(t *testing.T) { + resp, err := store.GetAnalyticsTopSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, tc.metric) + require.NoError(t, err, "GetAnalyticsTopSessions") + require.Len(t, resp.Sessions, 10, + "more than 10 model-matching sessions must cap at 10") + assert.Equal(t, "top-cap-11", resp.Sessions[0].ID, + "highest-count session ranks first") + assert.Equal(t, tc.topVal, tc.field(resp.Sessions[0]), + "top session filtered count") + }) + } +} + +func TestStoreGetAnalyticsVelocityModelFilterUsesMatchingRowsOnly( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_velocity_model_rows_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, message_count + ) VALUES ( + 'velocity-model-rows-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:11:00Z'::timestamptz, + 4 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('velocity-model-rows-001', 0, 'user', 'claude q', + '2024-06-01T09:00:00Z'::timestamptz, 8, FALSE, 'claude-3-5-sonnet'), + ('velocity-model-rows-001', 1, 'assistant', 'offscope-offscope-xx', + '2024-06-01T09:00:10Z'::timestamptz, 20, FALSE, 'claude-3-5-sonnet'), + ('velocity-model-rows-001', 2, 'user', 'gpt q', + '2024-06-01T09:10:00Z'::timestamptz, 5, FALSE, ''), + ('velocity-model-rows-001', 3, 'assistant', 'reply', + '2024-06-01T09:11:00Z'::timestamptz, 5, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO tool_calls ( + session_id, tool_name, category, call_index, message_ordinal + ) VALUES + ('velocity-model-rows-001', 'Read', 'Read', 0, 1), + ('velocity-model-rows-001', 'Bash', 'Bash', 1, 1), + ('velocity-model-rows-001', 'Grep', 'Grep', 2, 1), + ('velocity-model-rows-001', 'Edit', 'Edit', 0, 3), + ('velocity-model-rows-001', 'Write', 'Write', 1, 3)`) + require.NoError(t, err, "insert tool_calls") + + resp, err := store.GetAnalyticsVelocity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsVelocity") + assert.Equal(t, 60.0, resp.Overall.FirstResponseSec.P50, + "FirstResponse P50") + assert.Equal(t, 2.0, resp.Overall.MsgsPerActiveMin, + "MsgsPerActiveMin") + assert.Equal(t, 5.0, resp.Overall.CharsPerActiveMin, + "CharsPerActiveMin") + assert.Equal(t, 2.0, resp.Overall.ToolCallsPerActiveMin, + "ToolCallsPerActiveMin") +} + +func TestStoreGetAnalyticsVelocityModelFilterPreservesSubsecondTimestamps( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_velocity_model_subsecond_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, message_count + ) VALUES ( + 'velocity-model-subsecond-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:00:01Z'::timestamptz, + 2 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('velocity-model-subsecond-001', 0, 'user', 'gpt q', + '2024-06-01T09:00:00.000Z'::timestamptz, 5, FALSE, ''), + ('velocity-model-subsecond-001', 1, 'assistant', 'reply', + '2024-06-01T09:00:00.500Z'::timestamptz, 5, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsVelocity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsVelocity") + // The paired empty-model user turn at 09:00:00.000 and the gpt-4o assistant + // at 09:00:00.500 are 0.5s apart. A whole-second TO_CHAR in the scope + // resolver would collapse the first-response gap to 0.0s. + assert.Equal(t, 0.5, resp.Overall.FirstResponseSec.P50, + "FirstResponse P50 keeps the subsecond gap") +} + +func TestStoreGetAnalyticsVelocityModelFilterCountsNullTimestampToolCallsWithoutTimeFilter( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_velocity_model_null_tool_ts_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, message_count + ) VALUES ( + 'velocity-model-null-ts-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:11:00Z'::timestamptz, + 5 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('velocity-model-null-ts-001', 0, 'user', 'claude q', + '2024-06-01T09:00:00Z'::timestamptz, 8, FALSE, 'claude-3-5-sonnet'), + ('velocity-model-null-ts-001', 1, 'assistant', 'offscope-offscope-xx', + '2024-06-01T09:00:10Z'::timestamptz, 20, FALSE, 'claude-3-5-sonnet'), + ('velocity-model-null-ts-001', 2, 'user', 'gpt q', + '2024-06-01T09:10:00Z'::timestamptz, 5, FALSE, ''), + ('velocity-model-null-ts-001', 3, 'assistant', 'reply', + '2024-06-01T09:11:00Z'::timestamptz, 5, FALSE, 'gpt-4o'), + ('velocity-model-null-ts-001', 4, 'assistant', 'extra', + NULL, 5, FALSE, 'gpt-4o')`) + require.NoError(t, err, "insert messages") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO tool_calls ( + session_id, tool_name, category, call_index, message_ordinal + ) VALUES + ('velocity-model-null-ts-001', 'Read', 'Read', 0, 1), + ('velocity-model-null-ts-001', 'Bash', 'Bash', 1, 1), + ('velocity-model-null-ts-001', 'Grep', 'Grep', 2, 1), + ('velocity-model-null-ts-001', 'Edit', 'Edit', 0, 3), + ('velocity-model-null-ts-001', 'Write', 'Write', 1, 3), + ('velocity-model-null-ts-001', 'Search', 'Search', 0, 4)`) + require.NoError(t, err, "insert tool_calls") + + resp, err := store.GetAnalyticsVelocity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsVelocity") + assert.Equal(t, 60.0, resp.Overall.FirstResponseSec.P50, + "FirstResponse P50") + assert.Equal(t, 3.0, resp.Overall.ToolCallsPerActiveMin, + "ToolCallsPerActiveMin") +} + +func TestStoreGetAnalyticsSignalSessionsModelFilterKeepsParserUserEvidence( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_signal_model_parser_user_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count, + quality_signal_version, short_prompt_count + ) VALUES ( + 'signal-model-parser-user-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:10:00Z'::timestamptz, + 2, 1, + 1, 1 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model, has_tool_use + ) VALUES + ('signal-model-parser-user-001', 0, 'user', 'help', + '2024-06-01T09:00:00Z'::timestamptz, 4, FALSE, '', FALSE), + ('signal-model-parser-user-001', 1, 'assistant', 'reply', + '2024-06-01T09:01:00Z'::timestamptz, 5, FALSE, 'gpt-4o', TRUE)`) + require.NoError(t, err, "insert messages") + + resp, err := store.GetAnalyticsSignalSessions(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, "short_prompt_count", 10) + require.NoError(t, err, "GetAnalyticsSignalSessions") + require.Len(t, resp.Sessions, 1, "len(Sessions)") + assert.Equal(t, "help", resp.Sessions[0].Excerpt) + require.NotNil(t, resp.Sessions[0].MessageOrdinal) + assert.Equal(t, 0, *resp.Sessions[0].MessageOrdinal) +} + +func TestStoreGetAnalyticsSessionShapeModelFilterUsesMatchingRowsOnly( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_shape_model_rows_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, + message_count, user_message_count + ) VALUES ( + 'shape-model-rows-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:30:00Z'::timestamptz, + 6, 4 + )`) + require.NoError(t, err, "insert session") + + type row struct { + ordinal int + role string + content string + timestamp string + model string + hasToolUse bool + } + rows := []row{ + {0, "user", "gpt q", "2024-06-01T09:00:00Z", "", false}, + {1, "assistant", "gpt tool", "2024-06-01T09:01:00Z", "gpt-4o", true}, + {2, "user", "claude q1", "2024-06-01T09:02:00Z", "claude-3-5-sonnet", false}, + {3, "user", "claude q2", "2024-06-01T09:03:00Z", "claude-3-5-sonnet", false}, + {4, "user", "claude q3", "2024-06-01T09:04:00Z", "claude-3-5-sonnet", false}, + {5, "assistant", "claude reply", "2024-06-01T09:05:00Z", "claude-3-5-sonnet", false}, + } + for _, row := range rows { + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model, has_tool_use + ) VALUES ( + $1, $2, $3, $4, $5::timestamptz, + $6, FALSE, $7, $8 + )`, + "shape-model-rows-001", row.ordinal, row.role, row.content, + row.timestamp, len(row.content), row.model, row.hasToolUse, + ) + require.NoError(t, err, "insert message %d", row.ordinal) + } + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO tool_calls ( + session_id, tool_name, category, call_index, message_ordinal + ) VALUES ( + $1, $2, $3, $4, $5 + )`, + "shape-model-rows-001", "Read", "Read", 0, 1, + ) + require.NoError(t, err, "insert tool call") + + resp, err := store.GetAnalyticsSessionShape(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsSessionShape") + assert.Equal(t, 1, resp.Count, "Count") + + lengths := map[string]int{} + for _, bucket := range resp.LengthDistribution { + lengths[bucket.Label] = bucket.Count + } + assert.Equal(t, 1, lengths["1-5"], "filtered length bucket") + assert.Equal(t, 0, lengths["6-15"], "full-session count must not leak") + + autonomy := map[string]int{} + for _, bucket := range resp.AutonomyDistribution { + autonomy[bucket.Label] = bucket.Count + } + assert.Equal(t, 1, autonomy["1-2"], "filtered autonomy bucket") + assert.Equal(t, 0, autonomy["<0.5"], "off-model user turns must not leak") +} + +func TestStoreGetAnalyticsVelocityModelFilterUsesMatchingComplexityBucket( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_velocity_model_complexity_test") + ctx := context.Background() + + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, ended_at, message_count + ) VALUES ( + 'velocity-model-complexity-001', 'test-machine', 'alpha', 'mixed', + '2024-06-01T09:00:00Z'::timestamptz, + '2024-06-01T09:15:00Z'::timestamptz, + 16 + )`) + require.NoError(t, err, "insert session") + + type row struct { + ordinal int + role string + content string + timestamp string + model string + } + rows := []row{ + {0, "user", "gpt q", "2024-06-01T09:00:00Z", ""}, + {1, "assistant", "reply", "2024-06-01T09:01:00Z", "gpt-4o"}, + } + for i := 2; i < 16; i++ { + role := "user" + if i%2 == 1 { + role = "assistant" + } + rows = append(rows, row{ + ordinal: i, role: role, content: "claude", + timestamp: "2024-06-01T09:10:00Z", + model: "claude-3-5-sonnet", + }) + } + for _, row := range rows { + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES ( + $1, $2, $3, $4, $5::timestamptz, + $6, FALSE, $7 + )`, + "velocity-model-complexity-001", row.ordinal, row.role, + row.content, row.timestamp, len(row.content), row.model, + ) + require.NoError(t, err, "insert message %d", row.ordinal) + } + + resp, err := store.GetAnalyticsVelocity(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }) + require.NoError(t, err, "GetAnalyticsVelocity") + require.Len(t, resp.ByComplexity, 1, "len(ByComplexity)") + assert.Equal(t, "1-15", resp.ByComplexity[0].Label, + "complexity bucket should use filtered message count") + assert.Equal(t, 1, resp.ByComplexity[0].Sessions, "Sessions") +} + +func findPGHOWCell(cells []db.HourOfWeekCell, dow, hour int) int { + for _, cell := range cells { + if cell.DayOfWeek == dow && cell.Hour == hour { + return cell.Messages + } + } + return -1 +} diff --git a/internal/postgres/analytics_pgtest_test.go b/internal/postgres/analytics_pgtest_test.go index ef5ceffd3..4530c2884 100644 --- a/internal/postgres/analytics_pgtest_test.go +++ b/internal/postgres/analytics_pgtest_test.go @@ -1,6 +1,7 @@ package postgres import ( + "strings" "testing" "github.com/stretchr/testify/assert" @@ -9,6 +10,52 @@ import ( "go.kenn.io/agentsview/internal/db" ) +func TestBuildAnalyticsWhere_ModelFilter(t *testing.T) { + pb := ¶mBuilder{} + where := buildAnalyticsWhereWithDate( + db.AnalyticsFilter{ + From: "2024-06-01", + To: "2024-06-03", + Model: "gpt-4o, claude-3-5-sonnet", + }, + pgDateColS, + pb, + true, + "s.id", + ) + + assert.Contains(t, where, + "EXISTS (SELECT 1 FROM messages m WHERE m.session_id = s.id", + ) + assert.Contains(t, where, "m.model IN (") + assert.Len(t, pb.args, 4, "date range plus two model args") + assert.Equal(t, "gpt-4o", pb.args[2]) + assert.Equal(t, "claude-3-5-sonnet", pb.args[3]) +} + +func TestBuildAnalyticsWhere_ModelFilterTrimsEmptyValues(t *testing.T) { + pb := ¶mBuilder{} + where := buildAnalyticsWhereWithDate( + db.AnalyticsFilter{ + From: "2024-06-01", + To: "2024-06-03", + Model: " , gpt-4o , ", + }, + pgDateCol, + pb, + true, + "id", + ) + + assert.Contains(t, where, "m.model = ") + assert.NotContains(t, where, "m.model IN (") + assert.Equal(t, "gpt-4o", pb.args[2]) + assert.Equal(t, 1, + strings.Count(where, "m.model = "), + "expected a single model predicate", + ) +} + func TestRankTopSessions_DurationSort(t *testing.T) { sessions := []db.TopSession{ {ID: "a", ActiveDurationMin: 10.0}, diff --git a/internal/postgres/analytics_scope.go b/internal/postgres/analytics_scope.go new file mode 100644 index 000000000..ed9af7d14 --- /dev/null +++ b/internal/postgres/analytics_scope.go @@ -0,0 +1,163 @@ +package postgres + +import ( + "context" + "fmt" + "strings" + "time" + + "go.kenn.io/agentsview/internal/db" +) + +// messageScopeFilter adapts the model/day/hour parts of a db.AnalyticsFilter +// into the pure db.ScopeFilter. +func messageScopeFilter(f db.AnalyticsFilter) db.ScopeFilter { + models := make(map[string]struct{}) + for _, m := range csvFilterValues(f.Model) { + models[m] = struct{}{} + } + return db.ScopeFilter{ + Models: models, + DayOfWeek: f.DayOfWeek, + Hour: f.Hour, + } +} + +// messageScope holds the matched messages for a model-filtered analytics +// request, grouped by session. It is a pure value; all DB work happens during +// resolution. +type messageScope struct { + bySession map[string][]db.ScopedMessage +} + +// resolveAnalyticsMessageScope streams candidate messages for sessionIDs and +// reduces them to the model/time-matched set. It returns nil when no model +// filter is set, signalling the caller to keep its session-grain path. +// includeContent omits the (expensive) content column for count-only panels. +func (s *Store) resolveAnalyticsMessageScope( + ctx context.Context, + sessionIDs []string, + f db.AnalyticsFilter, + includeContent bool, +) (*messageScope, error) { + if strings.TrimSpace(f.Model) == "" { + return nil, nil + } + + seen := make(map[string]struct{}, len(sessionIDs)) + unique := make([]string, 0, len(sessionIDs)) + for _, id := range sessionIDs { + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + unique = append(unique, id) + } + + flt := messageScopeFilter(f) + loc := analyticsLocation(f) + bySession := make(map[string][]db.ScopedMessage, len(unique)) + emit := func(m db.ScopedMessage) { + bySession[m.SessionID] = append(bySession[m.SessionID], m) + } + + contentExpr := "''" + if includeContent { + contentExpr = "COALESCE(content, '')" + } + + if err := pgQueryChunked(unique, func(chunk []string) error { + reducer := db.NewScopeReducer(flt, emit) + pb := ¶mBuilder{} + placeholders := pgInPlaceholders(chunk, pb) + rows, err := s.pg.QueryContext(ctx, ` + SELECT session_id, ordinal, role, is_system, + COALESCE(model, ''), has_thinking, has_tool_use, + timestamp, + output_tokens, has_output_tokens, content_length, + `+contentExpr+` + FROM messages + WHERE session_id IN `+placeholders+` + ORDER BY session_id, ordinal`, + pb.args..., + ) + if err != nil { + return fmt.Errorf("querying analytics candidate messages: %w", err) + } + defer rows.Close() + + for rows.Next() { + var ( + sessionID, role, model, content string + ordinal, outputTokens, contentLength int + isSystem, hasThinking, hasToolUse, hasOutputTokens bool + ts *time.Time + ) + if err := rows.Scan( + &sessionID, &ordinal, &role, &isSystem, &model, + &hasThinking, &hasToolUse, &ts, &outputTokens, + &hasOutputTokens, &contentLength, &content, + ); err != nil { + return fmt.Errorf("scanning analytics candidate message: %w", err) + } + // Preserve subsecond precision: scan the timestamptz natively and + // format at full RFC3339Nano precision, matching the non-model + // velocity path and the SQLite/DuckDB scopes. A whole-second + // TO_CHAR here would truncate turn-cycle/first-response gaps. + tsStr := "" + if ts != nil { + tsStr = FormatISO8601(*ts) + } + parsed, has := localTime(tsStr, loc) + if err := reducer.Push(db.MessageInput{ + SessionID: sessionID, + Ordinal: ordinal, + Role: role, + Model: model, + IsSystem: isSystem, + Timestamp: tsStr, + LocalTime: parsed, + HasLocalTime: has, + HasThinking: hasThinking, + HasToolUse: hasToolUse, + OutputTokens: outputTokens, + HasOutputTokens: hasOutputTokens, + ContentLength: contentLength, + Content: content, + }); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating analytics candidate messages: %w", err) + } + return nil + }); err != nil { + return nil, err + } + + return &messageScope{bySession: bySession}, nil +} + +// MessagesBySession returns the matched rows per session. +func (s *messageScope) MessagesBySession() map[string][]db.ScopedMessage { + return s.bySession +} + +// StatsBySession aggregates matched rows per session. +func (s *messageScope) StatsBySession() map[string]db.MessageStats { + out := make(map[string]db.MessageStats, len(s.bySession)) + for id, rows := range s.bySession { + out[id] = db.ScopeStats(rows) + } + return out +} + +// TimingBySession projects matched rows into the velocity timing view. +func (s *messageScope) TimingBySession() map[string][]db.TimingMessage { + out := make(map[string][]db.TimingMessage, len(s.bySession)) + for id, rows := range s.bySession { + out[id] = db.ScopeTiming(rows) + } + return out +} diff --git a/internal/postgres/analytics_signals_pgtest_test.go b/internal/postgres/analytics_signals_pgtest_test.go index 92e74113b..fe5ffe898 100644 --- a/internal/postgres/analytics_signals_pgtest_test.go +++ b/internal/postgres/analytics_signals_pgtest_test.go @@ -101,3 +101,79 @@ func TestStoreGetAnalyticsSignals(t *testing.T) { assert.Equal(t, "proj", resp.ByProject[0].Project) assert.Equal(t, 2, resp.ByProject[0].SessionCount) } + +func TestStoreGetAnalyticsSignalSessionsModelFilterUsesMatchingMessages( + t *testing.T, +) { + pgURL := testPGURL(t) + cleanPGSchema(t, pgURL) + t.Cleanup(func() { cleanPGSchema(t, pgURL) }) + + local := testDB(t) + ps, err := New( + pgURL, "agentsview", local, + "signals-model-filter-machine", true, + SyncOptions{}, + ) + require.NoError(t, err, "creating sync") + defer ps.Close() + + ctx := context.Background() + require.NoError(t, ps.EnsureSchema(ctx), "ensure schema") + + started := "2024-06-01T09:00:00Z" + first := "tool evidence" + require.NoError(t, local.UpsertSession(db.Session{ + ID: "signal-mixed", + Project: "proj", + Machine: "local", + Agent: "claude", + FirstMessage: &first, + StartedAt: &started, + MessageCount: 2, + }), "upsert session") + require.NoError(t, local.InsertMessages([]db.Message{ + { + SessionID: "signal-mixed", Ordinal: 0, Role: "assistant", + Content: "claude tool evidence", ContentLength: 20, + Timestamp: "2024-06-01T09:05:00Z", + Model: "claude-3-5-sonnet", + HasToolUse: true, + }, + { + SessionID: "signal-mixed", Ordinal: 1, Role: "assistant", + Content: "gpt tool evidence", ContentLength: 17, + Timestamp: "2024-06-01T09:06:00Z", + Model: "gpt-4o", + HasToolUse: true, + }, + }), "insert messages") + require.NoError(t, local.UpdateSessionSignals( + "signal-mixed", + db.SessionSignalUpdate{ToolFailureSignalCount: 1}, + ), "update session signals") + + _, err = ps.Push(ctx, false, nil) + require.NoError(t, err, "push") + + store, err := NewStore(pgURL, "agentsview", true) + require.NoError(t, err, "NewStore") + defer store.Close() + + resp, err := store.GetAnalyticsSignalSessions( + ctx, + db.AnalyticsFilter{ + From: "2024-06-01", + To: "2024-06-01", + Timezone: "UTC", + Model: "gpt-4o", + }, + "tool_failure_signals", + 10, + ) + require.NoError(t, err, "GetAnalyticsSignalSessions") + require.Len(t, resp.Sessions, 1, "len(Sessions)") + assert.Equal(t, "gpt tool evidence", resp.Sessions[0].Excerpt) + require.NotNil(t, resp.Sessions[0].MessageOrdinal) + assert.Equal(t, 1, *resp.Sessions[0].MessageOrdinal) +} diff --git a/internal/postgres/analytics_unit_test.go b/internal/postgres/analytics_unit_test.go index ee7d96158..a8967c1eb 100644 --- a/internal/postgres/analytics_unit_test.go +++ b/internal/postgres/analytics_unit_test.go @@ -32,6 +32,7 @@ type analyticsProbeRows struct { type analyticsProbeState struct { mu sync.Mutex queries []string + args [][]any } var ( @@ -81,15 +82,59 @@ func (c *analyticsProbeConn) Begin() (driver.Tx, error) { } func (c *analyticsProbeConn) QueryContext( - _ context.Context, query string, _ []driver.NamedValue, + _ context.Context, query string, args []driver.NamedValue, ) (driver.Rows, error) { + values := make([]any, len(args)) + for i, arg := range args { + values[i] = arg.Value + } c.state.mu.Lock() c.state.queries = append(c.state.queries, query) + c.state.args = append(c.state.args, values) c.state.mu.Unlock() normalized := strings.ToLower(query) switch { + case strings.Contains(normalized, "select distinct model"): + sessionModels := make([][]driver.Value, 0) + for _, value := range values { + sessionID, ok := value.(string) + if !ok || !strings.HasPrefix(sessionID, "s") { + continue + } + sessionModels = append(sessionModels, + []driver.Value{"model-" + sessionID}, + ) + } + if len(sessionModels) == 0 { + sessionModels = [][]driver.Value{ + {"model-s1"}, + {"model-s2"}, + } + } + return &analyticsProbeRows{ + columns: []string{"model"}, + values: sessionModels, + }, nil case strings.Contains(normalized, "from sessions"): + if strings.Contains(normalized, "message_count, agent, project") { + return &analyticsProbeRows{ + columns: []string{ + "id", "date", "message_count", "agent", "project", + "total_output_tokens", "has_total_output_tokens", + }, + values: [][]driver.Value{ + { + "s1", time.Date(2024, 6, 3, 9, 0, 0, 0, time.UTC), + int64(10), "claude", "alpha", int64(0), false, + }, + { + "s2", time.Date(2024, 6, 4, 9, 0, 0, 0, time.UTC), + int64(20), "codex", "beta", int64(0), false, + }, + }, + }, nil + } if strings.Contains(normalized, "agent, project") { return &analyticsProbeRows{ columns: []string{"id", "date", "agent", "project"}, @@ -134,7 +179,32 @@ func (c *analyticsProbeConn) QueryContext( }, nil } if !strings.Contains(normalized, "group by session_id, category") { - return nil, errors.New("tool call query must group by session_id, category") + if !strings.Contains(normalized, + "group by tc.session_id, tc.category") { + return nil, errors.New( + "tool call query must group by session_id, category") + } + } + if strings.Contains(normalized, "to_char(m.timestamp") { + return &analyticsProbeRows{ + columns: []string{ + "session_id", "category", "count", "timestamp", + }, + values: [][]driver.Value{ + { + "s1", "Read", int64(2), + "2024-06-03T09:00:00Z", + }, + { + "s1", "Bash", int64(1), + "2024-06-03T09:00:00Z", + }, + { + "s2", "Read", int64(1), + "2024-06-04T09:00:00Z", + }, + }, + }, nil } return &analyticsProbeRows{ columns: []string{"session_id", "category", "count"}, @@ -237,6 +307,74 @@ func TestGetAnalyticsSkillsAggregatesToolCallsInSQL(t *testing.T) { "with session fallback for null timestamps") } +func TestGetAnalyticsToolsModelFilterJoinsMessages(t *testing.T) { + state := &analyticsProbeState{} + store := &Store{ + pg: newAnalyticsProbeDB(t, state), + } + + _, err := store.GetAnalyticsTools( + context.Background(), + db.AnalyticsFilter{ + From: "2024-06-01", + To: "2024-06-30", + Model: "gpt-4o", + }, + ) + require.NoError(t, err, "GetAnalyticsTools") + + state.mu.Lock() + defer state.mu.Unlock() + + var toolQuery string + for _, query := range state.queries { + if strings.Contains(strings.ToLower(query), "from tool_calls tc") { + toolQuery = query + break + } + } + require.NotEmpty(t, toolQuery, "tool query not captured") + normalized := strings.Join(strings.Fields(strings.ToLower(toolQuery)), " ") + assert.Contains(t, normalized, + "join messages m on m.session_id = tc.session_id and m.ordinal = tc.message_ordinal") + assert.Contains(t, normalized, "m.model = $") +} + +func TestGetAnalyticsSkillsModelFilterUsesMatchingMessages(t *testing.T) { + state := &analyticsProbeState{} + store := &Store{ + pg: newAnalyticsProbeDB(t, state), + } + + _, err := store.GetAnalyticsSkills( + context.Background(), + db.AnalyticsFilter{ + From: "2024-06-01", + To: "2024-06-30", + Model: "gpt-4o", + }, + ) + require.NoError(t, err, "GetAnalyticsSkills") + + state.mu.Lock() + defer state.mu.Unlock() + + var skillQuery string + for _, query := range state.queries { + q := strings.ToLower(query) + if strings.Contains(q, "from tool_calls tc") && + strings.Contains(q, "trim(coalesce(tc.skill_name") { + skillQuery = query + break + } + } + require.NotEmpty(t, skillQuery, "skill query not captured") + normalized := strings.Join(strings.Fields(strings.ToLower(skillQuery)), " ") + assert.Contains(t, normalized, + "left join messages m on m.session_id = tc.session_id and m.ordinal = tc.message_ordinal") + assert.Contains(t, normalized, "m.model = $") +} + func TestQueryVelocityMsgsScansNativeTimestamps(t *testing.T) { store := &Store{ pg: newAnalyticsProbeDB(t, &analyticsProbeState{}), @@ -257,3 +395,20 @@ func TestQueryVelocityMsgsScansNativeTimestamps(t *testing.T) { assert.Equal(t, 10.0, sessionMsgs["s1"][1].ts.Sub(sessionMsgs["s1"][0].ts).Seconds()) } + +func TestGetAnalyticsSummaryModelsFollowFilteredSessions(t *testing.T) { + store := &Store{ + pg: newAnalyticsProbeDB(t, &analyticsProbeState{}), + } + + resp, err := store.GetAnalyticsSummary( + context.Background(), + db.AnalyticsFilter{ + From: "2024-06-03", + To: "2024-06-03", + Timezone: "UTC", + }, + ) + require.NoError(t, err, "GetAnalyticsSummary") + assert.Equal(t, []string{"model-s1"}, resp.Models) +} diff --git a/internal/postgres/filter_test.go b/internal/postgres/filter_test.go index 190b7978e..8c43a80d8 100644 --- a/internal/postgres/filter_test.go +++ b/internal/postgres/filter_test.go @@ -31,6 +31,7 @@ func TestPGAutomatedScopePredicates(t *testing.T) { "created_at", ¶mBuilder{}, false, + "id", ) }, }, @@ -48,6 +49,7 @@ func TestPGAutomatedScopePredicates(t *testing.T) { "created_at", ¶mBuilder{}, false, + "id", ) }, }, @@ -65,6 +67,7 @@ func TestPGAutomatedScopePredicates(t *testing.T) { "created_at", ¶mBuilder{}, false, + "id", ) }, }, @@ -120,6 +123,7 @@ func TestPGAutomatedScopeOneShotExemption(t *testing.T) { "created_at", ¶mBuilder{}, false, + "id", ) want := "(user_message_count > 1 OR is_automated = TRUE)" assert.Contains(t, sql, want, "analytics SQL missing one-shot exemption") @@ -145,6 +149,7 @@ func TestPGAnalyticsMachineMultiSelectPredicate(t *testing.T) { "created_at", pb, false, + "id", ) want := "machine IN ($1,$2)" diff --git a/internal/postgres/trends.go b/internal/postgres/trends.go index ae5881e36..79ac20f90 100644 --- a/internal/postgres/trends.go +++ b/internal/postgres/trends.go @@ -3,6 +3,7 @@ package postgres import ( "context" "fmt" + "strings" "time" "go.kenn.io/agentsview/internal/db" @@ -31,9 +32,15 @@ func (s *Store) GetTrendsTerms( sessionFilter.To = "" sessionFilter.DayOfWeek = nil sessionFilter.Hour = nil + sessionFilter.Model = "" pb := ¶mBuilder{} - where := buildAnalyticsWhereWithoutDate(sessionFilter, pb) - query := `SELECT m.content, + preds := []string{buildAnalyticsWhereWithDate( + sessionFilter, "", pb, false, "s.id", + )} + flt := messageScopeFilter(f) + modelFiltering := len(flt.Models) > 0 + query := `SELECT m.session_id, m.ordinal, m.role, m.is_system, + COALESCE(m.model, ''), m.content, COALESCE(TO_CHAR(m.timestamp AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), ''), COALESCE(TO_CHAR(s.started_at AT TIME ZONE 'UTC', @@ -42,10 +49,11 @@ func (s *Store) GetTrendsTerms( 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), '') FROM sessions s JOIN messages m ON m.session_id = s.id - WHERE ` + where + ` + WHERE ` + strings.Join(preds, " AND ") + ` AND m.role IN ('user', 'assistant') AND m.is_system = FALSE - AND ` + db.PostgresSystemPrefixSQL("m.content", "m.role") + AND ` + db.PostgresSystemPrefixSQL("m.content", "m.role") + ` + ORDER BY m.session_id, m.ordinal` rows, err := s.pg.QueryContext(ctx, query, pb.args...) if err != nil { @@ -53,37 +61,86 @@ func (s *Store) GetTrendsTerms( } defer rows.Close() - for rows.Next() { - var content, msgTS, startedAt, createdAt string - if err := rows.Scan( - &content, &msgTS, &startedAt, &createdAt, - ); err != nil { - return db.TrendsTermsResponse{}, fmt.Errorf("scanning trends term row: %w", err) - } - msgTime, ok := trendMessageLocalTime(msgTS, startedAt, createdAt, loc) + type trendRow struct { + sessionID string + role string + isSystem bool + model string + content string + msgTS string + startedAt string + createdAt string + } + processRow := func(row trendRow) { + msgTime, ok := trendMessageLocalTime(row.msgTS, row.startedAt, row.createdAt, loc) if !ok { - continue - } - if f.HasTimeFilter() && !matchesTimeFilter(f, msgTime) { - continue + return } msgDate := msgTime.Format("2006-01-02") if !inDateRange(msgDate, f.From, f.To) { - continue + return } bucketDate := db.TrendBucketDate(msgTime, loc, granularity) bucket, ok := bucketIndex[bucketDate] if !ok { - continue + return } messageCounts[bucket]++ for i, term := range terms { - count := db.CountTrendOccurrences(content, term) + count := db.CountTrendOccurrences(row.content, term) if count > 0 { counts[i][bucket] += count } } } + rowStartedAt := make(map[string]string) + rowCreatedAt := make(map[string]string) + emit := func(m db.ScopedMessage) { + processRow(trendRow{ + sessionID: m.SessionID, + role: m.Role, + isSystem: m.IsSystem, + content: m.Content, + msgTS: m.Timestamp, + startedAt: rowStartedAt[m.SessionID], + createdAt: rowCreatedAt[m.SessionID], + }) + } + reducer := db.NewScopeReducer(flt, emit) + + for rows.Next() { + var row trendRow + var ordinal int + if err := rows.Scan( + &row.sessionID, &ordinal, &row.role, &row.isSystem, &row.model, + &row.content, &row.msgTS, &row.startedAt, &row.createdAt, + ); err != nil { + return db.TrendsTermsResponse{}, fmt.Errorf("scanning trends term row: %w", err) + } + if !modelFiltering { + msgTime, ok := trendMessageLocalTime(row.msgTS, row.startedAt, row.createdAt, loc) + if ok && flt.MatchesDayHour(msgTime, true) { + processRow(row) + } + continue + } + rowStartedAt[row.sessionID] = row.startedAt + rowCreatedAt[row.sessionID] = row.createdAt + msgTime, has := trendMessageLocalTime(row.msgTS, row.startedAt, row.createdAt, loc) + if err := reducer.Push(db.MessageInput{ + SessionID: row.sessionID, + Ordinal: ordinal, + Role: row.role, + Model: row.model, + IsSystem: row.isSystem, + Timestamp: row.msgTS, + LocalTime: msgTime, + HasLocalTime: has, + Content: row.content, + }); err != nil { + return db.TrendsTermsResponse{}, err + } + } if err := rows.Err(); err != nil { return db.TrendsTermsResponse{}, fmt.Errorf("iterating trends term rows: %w", err) } diff --git a/internal/postgres/trends_pgtest_test.go b/internal/postgres/trends_pgtest_test.go index 54cd922e3..ba89e15c5 100644 --- a/internal/postgres/trends_pgtest_test.go +++ b/internal/postgres/trends_pgtest_test.go @@ -97,6 +97,44 @@ func TestStoreGetTrendsTermsUsesMessageTimestampFilters(t *testing.T) { assert.Equal(t, 1, trendSeriesByTerm(got.Series)["seam"].Total) } +func TestStoreGetTrendsTermsModelFilterStaysOnMatchingMessages( + t *testing.T, +) { + _, store := prepareUsageSchema(t, "agentsview_trends_terms_model_messages_test") + ctx := context.Background() + _, err := store.DB().ExecContext(ctx, ` + INSERT INTO sessions ( + id, machine, project, agent, started_at, + message_count, user_message_count + ) VALUES ( + 'trends-pg-model-messages-001', 'test-machine', + 'alpha', 'claude', + '2024-06-01T09:00:00Z'::timestamptz, 3, 1 + )`) + require.NoError(t, err, "insert session") + _, err = store.DB().ExecContext(ctx, ` + INSERT INTO messages ( + session_id, ordinal, role, content, timestamp, + content_length, is_system, model + ) VALUES + ('trends-pg-model-messages-001', 0, 'user', 'seam', + '2024-06-01T09:00:00Z'::timestamptz, 4, FALSE, ''), + ('trends-pg-model-messages-001', 1, 'assistant', 'ready', + '2024-06-01T09:01:00Z'::timestamptz, 5, FALSE, 'gpt-4o'), + ('trends-pg-model-messages-001', 2, 'assistant', 'seam seam', + '2024-06-01T09:05:00Z'::timestamptz, 9, FALSE, 'claude-3-5-sonnet')`) + require.NoError(t, err, "insert messages") + terms, err := db.ParseTrendTerms([]string{"seam"}) + require.NoError(t, err, "ParseTrendTerms") + got, err := store.GetTrendsTerms(ctx, db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-01", Timezone: "UTC", + Model: "gpt-4o", + }, terms, "day") + require.NoError(t, err, "GetTrendsTerms") + assert.Equal(t, 2, got.MessageCount) + assert.Equal(t, 1, trendSeriesByTerm(got.Series)["seam"].Total) +} + func trendBucketDates(buckets []db.TrendBucket) []string { dates := make([]string, len(buckets)) for i, bucket := range buckets { diff --git a/internal/postgres/trends_unit_test.go b/internal/postgres/trends_unit_test.go new file mode 100644 index 000000000..6f9462f70 --- /dev/null +++ b/internal/postgres/trends_unit_test.go @@ -0,0 +1,134 @@ +package postgres + +import ( + "context" + "database/sql" + "database/sql/driver" + "io" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +type trendsProbeDriver struct{} + +type trendsProbeConn struct { + state *trendsProbeState +} + +type trendsProbeRows struct { + columns []string + values [][]driver.Value + next int +} + +type trendsProbeState struct { + mu sync.Mutex + queries []string +} + +var ( + trendsProbeRegisterOnce sync.Once + trendsProbeStatesMu sync.Mutex + trendsProbeStates = map[string]*trendsProbeState{} +) + +func newTrendsProbeDB( + t *testing.T, state *trendsProbeState, +) *sql.DB { + t.Helper() + trendsProbeRegisterOnce.Do(func() { + sql.Register("agentsview_trends_probe", trendsProbeDriver{}) + }) + name := t.Name() + trendsProbeStatesMu.Lock() + trendsProbeStates[name] = state + trendsProbeStatesMu.Unlock() + t.Cleanup(func() { + trendsProbeStatesMu.Lock() + delete(trendsProbeStates, name) + trendsProbeStatesMu.Unlock() + }) + + pg, err := sql.Open("agentsview_trends_probe", name) + require.NoError(t, err, "open trends probe db") + t.Cleanup(func() { pg.Close() }) + return pg +} + +func (trendsProbeDriver) Open(name string) (driver.Conn, error) { + trendsProbeStatesMu.Lock() + state := trendsProbeStates[name] + trendsProbeStatesMu.Unlock() + return &trendsProbeConn{state: state}, nil +} + +func (c *trendsProbeConn) Prepare(string) (driver.Stmt, error) { + return nil, driver.ErrSkip +} + +func (c *trendsProbeConn) Close() error { return nil } + +func (c *trendsProbeConn) Begin() (driver.Tx, error) { + return nil, driver.ErrSkip +} + +func (c *trendsProbeConn) QueryContext( + _ context.Context, + query string, + _ []driver.NamedValue, +) (driver.Rows, error) { + c.state.mu.Lock() + c.state.queries = append(c.state.queries, query) + c.state.mu.Unlock() + return &trendsProbeRows{ + columns: []string{"content", "timestamp", "started_at", "created_at"}, + values: nil, + }, nil +} + +func (r *trendsProbeRows) Columns() []string { return r.columns } + +func (r *trendsProbeRows) Close() error { return nil } + +func (r *trendsProbeRows) Next(dest []driver.Value) error { + if r.next >= len(r.values) { + return io.EOF + } + copy(dest, r.values[r.next]) + r.next++ + return nil +} + +func TestGetTrendsTermsModelFilterTargetsOuterMessages(t *testing.T) { + state := &trendsProbeState{} + store := &Store{ + pg: newTrendsProbeDB(t, state), + } + terms, err := db.ParseTrendTerms([]string{"seam"}) + require.NoError(t, err, "ParseTrendTerms") + + _, err = store.GetTrendsTerms( + context.Background(), + db.AnalyticsFilter{ + From: "2024-06-01", To: "2024-06-02", + Timezone: "UTC", + Model: "gpt-4o", + }, + terms, + "day", + ) + require.NoError(t, err, "GetTrendsTerms") + require.NotEmpty(t, state.queries, "queries") + + query := strings.ToLower(strings.Join(state.queries, "\n")) + assert.Contains(t, query, "join messages m on m.session_id = s.id") + assert.Contains(t, query, "order by m.session_id, m.ordinal") + assert.NotContains(t, query, "and m.model = $1") + assert.NotContains(t, query, "exists (select 1 from messages") +} diff --git a/internal/server/analytics_test.go b/internal/server/analytics_test.go index 54756605f..9c78ded38 100644 --- a/internal/server/analytics_test.go +++ b/internal/server/analytics_test.go @@ -35,13 +35,13 @@ func seedAnalyticsEnv(t *testing.T, te *testEnv) seedStats { t.Helper() type entry struct { - id, project, agent, started string - msgs int + id, project, agent, started, model string + msgs int } entries := []entry{ - {"a1", "alpha", "claude", "2024-06-01T09:00:00Z", 10}, - {"a2", "alpha", "codex", "2024-06-01T14:00:00Z", 20}, - {"b1", "beta", "claude", "2024-06-02T10:00:00Z", 30}, + {"a1", "alpha", "claude", "2024-06-01T09:00:00Z", "claude-3-5-sonnet", 10}, + {"a2", "alpha", "codex", "2024-06-01T14:00:00Z", "gpt-4o", 20}, + {"b1", "beta", "claude", "2024-06-02T10:00:00Z", "claude-3-5-sonnet", 30}, } stats := seedStats{ @@ -74,6 +74,7 @@ func seedAnalyticsEnv(t *testing.T, te *testEnv) seedStats { // Skill analytics now buckets and filters by message // timestamp, so align messages with the session window. m.Timestamp = started + m.Model = s.model // Add tool calls on every other assistant msg if m.Role == "assistant" && i%4 == 1 { m.HasToolUse = true @@ -194,6 +195,23 @@ func TestAnalyticsSummary(t *testing.T) { assert.Equal(t, stats.TotalMessages, resp.TotalMessages) assert.Equal(t, stats.ActiveProjects, resp.ActiveProjects) assert.Equal(t, stats.ActiveDays, resp.ActiveDays) + assert.Equal(t, + []string{"claude-3-5-sonnet", "gpt-4o"}, + resp.Models, + ) + }) + + t.Run("ModelFilter", func(t *testing.T) { + w := te.get(t, buildURLWithRange("summary", map[string]string{ + "timezone": "UTC", + "model": "gpt-4o", + })) + assertStatus(t, w, http.StatusOK) + + resp := decode[db.AnalyticsSummary](t, w) + assert.Equal(t, 1, resp.TotalSessions) + assert.Equal(t, 20, resp.TotalMessages) + assert.Equal(t, []string{"gpt-4o"}, resp.Models) }) t.Run("NonUTCTimezone", func(t *testing.T) { diff --git a/internal/server/huma_routes_analytics.go b/internal/server/huma_routes_analytics.go index 869821b20..60f3d6493 100644 --- a/internal/server/huma_routes_analytics.go +++ b/internal/server/huma_routes_analytics.go @@ -40,6 +40,7 @@ type AnalyticsFilterInput struct { Machine string `query:"machine" doc:"Filter by machine"` Project string `query:"project" doc:"Filter by project"` Agent string `query:"agent" doc:"Filter by agent"` + Model string `query:"model" doc:"Comma-separated model filter"` DayOfWeek optionalIntParam `query:"dow" minimum:"0" maximum:"6" doc:"Day of week, Monday=0 through Sunday=6"` Hour optionalIntParam `query:"hour" minimum:"0" maximum:"23" doc:"Hour of day, 0 through 23"` MinUserMessages int `query:"min_user_messages" minimum:"0" doc:"Minimum user message count"` @@ -95,6 +96,7 @@ func analyticsFilterFromInput(in AnalyticsFilterInput) (db.AnalyticsFilter, erro Machine: in.Machine, Project: in.Project, Agent: in.Agent, + Model: in.Model, Timezone: tz, DayOfWeek: optionalIntValue(in.DayOfWeek), Hour: optionalIntValue(in.Hour),