diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/app/App.svelte | 12 | ||||
| -rw-r--r-- | src/core/chunks/reducer.ts | 4 | ||||
| -rw-r--r-- | src/core/telemetry/index.ts | 14 | ||||
| -rw-r--r-- | src/core/telemetry/reducer.test.ts | 252 | ||||
| -rw-r--r-- | src/core/telemetry/reducer.ts | 122 | ||||
| -rw-r--r-- | src/core/telemetry/selectors.ts | 95 | ||||
| -rw-r--r-- | src/core/telemetry/types.ts | 35 | ||||
| -rw-r--r-- | src/core/wire/conformance.test.ts | 14 | ||||
| -rw-r--r-- | src/core/wire/conformance.ts | 2 | ||||
| -rw-r--r-- | src/features/chat/index.ts | 2 | ||||
| -rw-r--r-- | src/features/chat/store.svelte.ts | 12 | ||||
| -rw-r--r-- | src/features/chat/store.test.ts | 46 | ||||
| -rw-r--r-- | src/features/chat/ui.test.ts | 150 | ||||
| -rw-r--r-- | src/features/chat/ui/ChatView.svelte | 93 | ||||
| -rw-r--r-- | src/features/chat/ui/TurnSummary.svelte | 75 |
15 files changed, 887 insertions, 41 deletions
diff --git a/src/app/App.svelte b/src/app/App.svelte index 61b4cb9..e1d59f9 100644 --- a/src/app/App.svelte +++ b/src/app/App.svelte @@ -1,6 +1,6 @@ <script lang="ts"> import type { InvokeMessage } from "@dispatch/ui-contract"; - import { ChatView, Composer, ModelSelector } from "../features/chat"; + import { ChatView, Composer, ModelSelector, TurnSummary } from "../features/chat"; import { TabBar } from "../features/tabs"; import { SurfaceView } from "../features/surface-host"; import type { AppStore } from "./store.svelte"; @@ -62,7 +62,15 @@ <div class="flex-1 overflow-y-auto"> {#key store.activeConversationId} - <ChatView chunks={store.activeChat.chunks} /> + <ChatView + chunks={store.activeChat.chunks} + telemetry={store.activeChat.telemetry} + currentTurnId={store.activeChat.currentTurnId} + /> + <TurnSummary + telemetry={store.activeChat.telemetry} + turnId={store.activeChat.currentTurnId} + /> {/key} </div> diff --git a/src/core/chunks/reducer.ts b/src/core/chunks/reducer.ts index 1dcfa39..54b1922 100644 --- a/src/core/chunks/reducer.ts +++ b/src/core/chunks/reducer.ts @@ -148,6 +148,10 @@ export function foldEvent(state: TranscriptState, event: AgentEvent): Transcript case "usage": return { ...state, latestUsage: event.usage }; + case "step-complete": + // Timing metadata — no content chunk; handled by the telemetry reducer. + return state; + case "done": { const provisional = flushAccumulating(state.provisional, state.accumulating); return { diff --git a/src/core/telemetry/index.ts b/src/core/telemetry/index.ts new file mode 100644 index 0000000..a528b0d --- /dev/null +++ b/src/core/telemetry/index.ts @@ -0,0 +1,14 @@ +export { foldMetricEvent, initialState } from "./reducer"; +export { + stepCount, + stepMetrics, + stepToolDuration, + stepTps, + totalDecodeMs, + totalInputTokens, + totalOutputTokens, + turnMetrics, + turnTps, + turnTtft, +} from "./selectors"; +export type { StepMetrics, TelemetryState, TurnMetrics } from "./types"; diff --git a/src/core/telemetry/reducer.test.ts b/src/core/telemetry/reducer.test.ts new file mode 100644 index 0000000..119bf96 --- /dev/null +++ b/src/core/telemetry/reducer.test.ts @@ -0,0 +1,252 @@ +import type { StepId, Usage } from "@dispatch/wire"; +import { describe, expect, it } from "vitest"; +import { foldMetricEvent, initialState } from "./reducer"; +import { + stepCount, + stepMetrics, + stepToolDuration, + stepTps, + totalDecodeMs, + totalInputTokens, + totalOutputTokens, + turnMetrics, + turnTps, + turnTtft, +} from "./selectors"; + +const sid = (s: string) => s as StepId; + +const usage = (turnId: string, stepId: string, u: Usage) => ({ + type: "usage" as const, + conversationId: "c1", + turnId, + stepId: sid(stepId), + usage: u, +}); + +const stepComplete = ( + turnId: string, + stepId: string, + timing: { ttftMs?: number; decodeMs?: number; genTotalMs?: number }, +) => ({ + type: "step-complete" as const, + conversationId: "c1", + turnId, + stepId: sid(stepId), + ...timing, +}); + +describe("foldMetricEvent", () => { + it("turn-start initializes an empty turn", () => { + const s = foldMetricEvent(initialState(), { + type: "turn-start", + conversationId: "c1", + turnId: "t1", + }); + expect(s.turns.get("t1")?.steps).toEqual([]); + }); + + it("step-complete populates timing on a new step", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent( + s, + stepComplete("t1", "s0", { ttftMs: 300, decodeMs: 800, genTotalMs: 1100 }), + ); + + const step = stepMetrics(s, "t1", 0); + expect(step?.ttftMs).toBe(300); + expect(step?.decodeMs).toBe(800); + expect(step?.genTotalMs).toBe(1100); + }); + + it("usage merges tokens into a step (joined by stepId)", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, stepComplete("t1", "s0", { genTotalMs: 500 })); + s = foldMetricEvent(s, usage("t1", "s0", { inputTokens: 100, outputTokens: 50 })); + + const step = stepMetrics(s, "t1", 0); + expect(step?.usage?.inputTokens).toBe(100); + expect(step?.usage?.outputTokens).toBe(50); + expect(step?.genTotalMs).toBe(500); // timing preserved + }); + + it("usage without stepId is ignored", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, { + type: "usage", + conversationId: "c1", + turnId: "t1", + usage: { inputTokens: 100, outputTokens: 50 }, + // no stepId + }); + expect(s.turns.get("t1")?.steps).toEqual([]); + }); + + it("tool-result accumulates durationMs into its step", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, stepComplete("t1", "s0", {})); + s = foldMetricEvent(s, { + type: "tool-result", + conversationId: "c1", + turnId: "t1", + stepId: sid("s0"), + toolCallId: "tc1", + toolName: "bash", + content: "", + isError: false, + durationMs: 120, + }); + s = foldMetricEvent(s, { + type: "tool-result", + conversationId: "c1", + turnId: "t1", + stepId: sid("s0"), + toolCallId: "tc2", + toolName: "bash", + content: "", + isError: false, + durationMs: 80, + }); + + const step = stepMetrics(s, "t1", 0); + expect(step?.toolDurationMs).toBe(200); + }); + + it("done records turn wall-clock and aggregate usage", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, { + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "complete", + durationMs: 4200, + usage: { inputTokens: 800, outputTokens: 200 }, + }); + + const turn = turnMetrics(s, "t1"); + expect(turn?.wallMs).toBe(4200); + expect(turn?.doneUsage?.outputTokens).toBe(200); + }); + + it("events for an unknown turn are handled gracefully (step-complete, usage)", () => { + const s = initialState(); + // step-complete for a turn we haven't started — creates the turn. + const s2 = foldMetricEvent(s, stepComplete("t1", "s0", { ttftMs: 100 })); + expect(s2.turns.get("t1")?.steps[0]?.ttftMs).toBe(100); + }); + + it("multiple steps accumulate in order", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, stepComplete("t1", "s0", { genTotalMs: 100 })); + s = foldMetricEvent(s, stepComplete("t1", "s1", { genTotalMs: 200 })); + + expect(stepCount(s, "t1")).toBe(2); + expect(stepMetrics(s, "t1", 0)?.genTotalMs).toBe(100); + expect(stepMetrics(s, "t1", 1)?.genTotalMs).toBe(200); + }); + + it("non-metric events are no-ops", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, { + type: "text-delta", + conversationId: "c1", + turnId: "t1", + delta: "hi", + }); + s = foldMetricEvent(s, { + type: "turn-sealed", + conversationId: "c1", + turnId: "t1", + }); + expect(s.turns.get("t1")?.steps).toEqual([]); + }); +}); + +describe("selectors — derived metrics", () => { + function populatedState() { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent( + s, + stepComplete("t1", "s0", { ttftMs: 300, decodeMs: 700, genTotalMs: 1000 }), + ); + s = foldMetricEvent(s, usage("t1", "s0", { inputTokens: 500, outputTokens: 100 })); + s = foldMetricEvent( + s, + stepComplete("t1", "s1", { ttftMs: 200, decodeMs: 500, genTotalMs: 700 }), + ); + s = foldMetricEvent(s, usage("t1", "s1", { inputTokens: 600, outputTokens: 80 })); + s = foldMetricEvent(s, { + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "complete", + durationMs: 3500, + usage: { inputTokens: 1100, outputTokens: 180 }, + }); + return s; + } + + it("stepTps = outputTokens / (decodeMs / 1000)", () => { + const s = populatedState(); + const step = stepMetrics(s, "t1", 0)!; + expect(stepTps(step)).toBeCloseTo(100 / 0.7, 2); + }); + + it("turnTtft returns first step's ttftMs", () => { + expect(turnTtft(populatedState(), "t1")).toBe(300); + }); + + it("totalDecodeMs sums all steps' decodeMs", () => { + expect(totalDecodeMs(populatedState(), "t1")).toBe(1200); + }); + + it("turnTps = outputTokens / (totalDecodeMs / 1000)", () => { + const s = populatedState(); + expect(turnTps(s, "t1")).toBeCloseTo(180 / 1.2, 2); + }); + + it("totalOutputTokens prefers done.usage over step sum", () => { + const s = populatedState(); + expect(totalOutputTokens(s, "t1")).toBe(180); // from done.usage + }); + + it("totalInputTokens prefers done.usage over step sum", () => { + const s = populatedState(); + expect(totalInputTokens(s, "t1")).toBe(1100); + }); + + it("stepToolDuration returns sum only when > 0", () => { + const withTools = foldMetricEvent( + foldMetricEvent(initialState(), { type: "turn-start", conversationId: "c1", turnId: "t1" }), + { + type: "tool-result", + conversationId: "c1", + turnId: "t1", + stepId: sid("s0"), + toolCallId: "tc1", + toolName: "bash", + content: "", + isError: false, + durationMs: 50, + }, + ); + const step = stepMetrics(withTools, "t1", 0)!; + expect(stepToolDuration(step)).toBe(50); + expect(stepToolDuration({ stepId: sid("s0") })).toBeUndefined(); + }); + + it("returns undefined for absent fields gracefully", () => { + const s = initialState(); + expect(turnMetrics(s, "missing")).toBeUndefined(); + expect(turnTtft(s, "missing")).toBeUndefined(); + expect(turnTps(s, "missing")).toBeUndefined(); + }); +}); diff --git a/src/core/telemetry/reducer.ts b/src/core/telemetry/reducer.ts new file mode 100644 index 0000000..4083231 --- /dev/null +++ b/src/core/telemetry/reducer.ts @@ -0,0 +1,122 @@ +import type { AgentEvent, StepId, Usage } from "@dispatch/wire"; +import type { StepMetrics, TelemetryState, TurnMetrics } from "./types"; + +/** The initial empty telemetry state. */ +export function initialState(): TelemetryState { + return { turns: new Map() }; +} + +function mergeStep(existing: StepMetrics, patch: StepMetrics): StepMetrics { + const merged: StepMetrics = { ...existing }; + if (patch.ttftMs !== undefined) (merged as { ttftMs?: number }).ttftMs = patch.ttftMs; + if (patch.decodeMs !== undefined) (merged as { decodeMs?: number }).decodeMs = patch.decodeMs; + if (patch.genTotalMs !== undefined) + (merged as { genTotalMs?: number }).genTotalMs = patch.genTotalMs; + if (patch.usage !== undefined) { + (merged as { usage?: Usage }).usage = { ...existing.usage, ...patch.usage }; + } + if (patch.toolDurationMs !== undefined) { + (merged as { toolDurationMs?: number }).toolDurationMs = + (existing.toolDurationMs ?? 0) + patch.toolDurationMs; + } + return merged; +} + +function upsertStep( + steps: readonly StepMetrics[], + stepId: StepId, + patch: StepMetrics, +): readonly StepMetrics[] { + const idx = steps.findIndex((s) => s.stepId === stepId); + if (idx === -1) { + return [...steps, patch]; + } + return [...steps.slice(0, idx), mergeStep(steps[idx]!, patch), ...steps.slice(idx + 1)]; +} + +function setTurn( + turns: ReadonlyMap<string, TurnMetrics>, + turnId: string, + turn: TurnMetrics, +): ReadonlyMap<string, TurnMetrics> { + const next = new Map(turns); + next.set(turnId, turn); + return next; +} + +/** + * Fold one live AgentEvent into the telemetry state. + * + * - `turn-start` records the active turnId. + * - `step-complete` creates/updates the step's timing metrics. + * - `usage` merges token counts into the step (joined by `stepId`). + * - `tool-result` accumulates `durationMs` into the step. + * - `done` records turn-level wall-clock + token totals. + * - All other event types are no-ops (content events belong to the transcript). + * + * Pure: input → output, no DOM, no side effects. + */ +export function foldMetricEvent(state: TelemetryState, event: AgentEvent): TelemetryState { + switch (event.type) { + case "turn-start": { + return { + ...state, + turns: setTurn(state.turns, event.turnId, { steps: [] }), + }; + } + + case "step-complete": { + const turnId = event.turnId; + const existing = state.turns.get(turnId); + const patch: StepMetrics = { stepId: event.stepId }; + if (event.ttftMs !== undefined) (patch as { ttftMs?: number }).ttftMs = event.ttftMs; + if (event.decodeMs !== undefined) (patch as { decodeMs?: number }).decodeMs = event.decodeMs; + if (event.genTotalMs !== undefined) + (patch as { genTotalMs?: number }).genTotalMs = event.genTotalMs; + const steps = + existing !== undefined ? upsertStep(existing.steps, event.stepId, patch) : [patch]; + return { + ...state, + turns: setTurn(state.turns, turnId, { ...existing, steps } as TurnMetrics), + }; + } + + case "usage": { + if (event.stepId === undefined) return state; + const turnId = event.turnId; + const existing = state.turns.get(turnId); + const patch: StepMetrics = { stepId: event.stepId, usage: event.usage }; + const steps = + existing !== undefined ? upsertStep(existing.steps, event.stepId, patch) : [patch]; + return { + ...state, + turns: setTurn(state.turns, turnId, { ...existing, steps } as TurnMetrics), + }; + } + + case "tool-result": { + if (event.durationMs === undefined) return state; + const turnId = event.turnId; + const existing = state.turns.get(turnId); + if (existing === undefined) return state; + const patch: StepMetrics = { stepId: event.stepId, toolDurationMs: event.durationMs }; + const steps = upsertStep(existing.steps, event.stepId, patch); + return { ...state, turns: setTurn(state.turns, turnId, { ...existing, steps }) }; + } + + case "done": { + const turnId = event.turnId; + const existing = state.turns.get(turnId); + const updated: TurnMetrics = { + ...(existing ?? { steps: [] }), + }; + if (event.durationMs !== undefined) + (updated as { wallMs?: number }).wallMs = event.durationMs; + if (event.usage !== undefined) (updated as { doneUsage?: Usage }).doneUsage = event.usage; + return { ...state, turns: setTurn(state.turns, turnId, updated) }; + } + + default: + return state; + } +} diff --git a/src/core/telemetry/selectors.ts b/src/core/telemetry/selectors.ts new file mode 100644 index 0000000..ecf1794 --- /dev/null +++ b/src/core/telemetry/selectors.ts @@ -0,0 +1,95 @@ +import type { Usage } from "@dispatch/wire"; +import type { StepMetrics, TelemetryState, TurnMetrics } from "./types"; + +/** Get the metrics for a specific step within a turn. */ +export function stepMetrics( + state: TelemetryState, + turnId: string, + stepIndex: number, +): StepMetrics | undefined { + return state.turns.get(turnId)?.steps[stepIndex]; +} + +/** Get the metrics for a turn. */ +export function turnMetrics(state: TelemetryState, turnId: string): TurnMetrics | undefined { + return state.turns.get(turnId); +} + +/** The number of steps in a turn. */ +export function stepCount(state: TelemetryState, turnId: string): number { + return state.turns.get(turnId)?.steps.length ?? 0; +} + +/** TTFT of the first step in a turn (the turn-visible first-token latency). */ +export function turnTtft(state: TelemetryState, turnId: string): number | undefined { + return state.turns.get(turnId)?.steps[0]?.ttftMs; +} + +/** Sum of all steps' decode times in a turn. */ +export function totalDecodeMs(state: TelemetryState, turnId: string): number | undefined { + const steps = state.turns.get(turnId)?.steps; + if (steps === undefined || steps.length === 0) return undefined; + let total = 0; + let found = false; + for (const s of steps) { + if (s.decodeMs !== undefined) { + total += s.decodeMs; + found = true; + } + } + return found ? total : undefined; +} + +/** Aggregate output tokens across all steps in a turn. */ +export function totalOutputTokens(state: TelemetryState, turnId: string): number | undefined { + const turn = state.turns.get(turnId); + if (turn === undefined) return undefined; + if (turn.doneUsage !== undefined) return turn.doneUsage.outputTokens; + let total = 0; + let found = false; + for (const s of turn.steps) { + if (s.usage?.outputTokens !== undefined) { + total += s.usage.outputTokens; + found = true; + } + } + return found ? total : undefined; +} + +/** Aggregate input tokens across all steps in a turn. */ +export function totalInputTokens(state: TelemetryState, turnId: string): number | undefined { + const turn = state.turns.get(turnId); + if (turn === undefined) return undefined; + if (turn.doneUsage !== undefined) return turn.doneUsage.inputTokens; + let total = 0; + let found = false; + for (const s of turn.steps) { + if (s.usage?.inputTokens !== undefined) { + total += s.usage.inputTokens; + found = true; + } + } + return found ? total : undefined; +} + +/** Derived TPS for a step: outputTokens / (decodeMs / 1000). */ +export function stepTps(step: StepMetrics): number | undefined { + if (step.usage?.outputTokens === undefined || step.decodeMs === undefined) return undefined; + if (step.decodeMs === 0) return undefined; + return step.usage.outputTokens / (step.decodeMs / 1000); +} + +/** Derived aggregate TPS for a turn. */ +export function turnTps(state: TelemetryState, turnId: string): number | undefined { + const outTokens = totalOutputTokens(state, turnId); + const decode = totalDecodeMs(state, turnId); + if (outTokens === undefined || decode === undefined || decode === 0) return undefined; + return outTokens / (decode / 1000); +} + +/** Sum of tool execution durations within a step. */ +export function stepToolDuration(step: StepMetrics): number | undefined { + return step.toolDurationMs !== undefined && step.toolDurationMs > 0 + ? step.toolDurationMs + : undefined; +} diff --git a/src/core/telemetry/types.ts b/src/core/telemetry/types.ts new file mode 100644 index 0000000..395ec93 --- /dev/null +++ b/src/core/telemetry/types.ts @@ -0,0 +1,35 @@ +import type { StepId, Usage } from "@dispatch/wire"; + +/** + * Per-step metrics, accumulated from `step-complete` + `usage` events. + * All fields optional — absent when the backend had no clock or the step + * produced no text/reasoning token. + */ +export interface StepMetrics { + readonly stepId: StepId; + readonly ttftMs?: number; + readonly decodeMs?: number; + readonly genTotalMs?: number; + readonly usage?: Usage; + readonly toolDurationMs?: number; // sum of tool-result.durationMs in this step +} + +/** + * Per-turn metrics, accumulated from `done` events + per-step aggregation. + */ +export interface TurnMetrics { + readonly wallMs?: number; + readonly doneUsage?: Usage; + readonly steps: readonly StepMetrics[]; +} + +/** + * Pure telemetry state — lives alongside but separate from TranscriptState. + * Accumulates live-only metric events; never persisted (history has no metrics). + * No "active turn" tracking — the consumer (store) passes the relevant turnId + * to the selectors. Pure: events flow in, derived values flow out. + */ +export interface TelemetryState { + /** turnId → TurnMetrics. Multiple turns accumulate (tab switching). */ + readonly turns: ReadonlyMap<string, TurnMetrics>; +} diff --git a/src/core/wire/conformance.test.ts b/src/core/wire/conformance.test.ts index 50b7f35..690ba4e 100644 --- a/src/core/wire/conformance.test.ts +++ b/src/core/wire/conformance.test.ts @@ -62,6 +62,15 @@ describe("classifies every AgentEvent type", () => { turnId: "t1", usage: { inputTokens: 10, outputTokens: 20 }, }, + { + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: "t1#0" as StepId, + ttftMs: 300, + decodeMs: 700, + genTotalMs: 1000, + }, { type: "error", conversationId: "c1", turnId: "t1", message: "oops" }, { type: "done", conversationId: "c1", turnId: "t1", reason: "complete" }, { type: "turn-sealed", conversationId: "c1", turnId: "t1" }, @@ -78,14 +87,15 @@ describe("classifies every AgentEvent type", () => { "tool-result", "tool-output", "usage", + "step-complete", "error", "done", "turn-sealed", ]); }); - it("covers all 11 AgentEvent variants", () => { - expect(samples).toHaveLength(11); + it("covers all 12 AgentEvent variants", () => { + expect(samples).toHaveLength(12); }); }); diff --git a/src/core/wire/conformance.ts b/src/core/wire/conformance.ts index 5d75a60..d89772e 100644 --- a/src/core/wire/conformance.ts +++ b/src/core/wire/conformance.ts @@ -30,6 +30,8 @@ export function assertAgentEventExhaustive(event: AgentEvent): string { return "done"; case "turn-sealed": return "turn-sealed"; + case "step-complete": + return "step-complete"; default: return event satisfies never; } diff --git a/src/features/chat/index.ts b/src/features/chat/index.ts index 4f2091a..b096cca 100644 --- a/src/features/chat/index.ts +++ b/src/features/chat/index.ts @@ -1,8 +1,10 @@ export type { RenderedChunk, RenderGroup, ToolBatchEntry } from "../../core/chunks"; export { groupRenderedChunks } from "../../core/chunks"; +export type { StepMetrics, TelemetryState, TurnMetrics } from "../../core/telemetry"; export type { ChatTransport, HistorySync } from "./ports"; export type { ChatStore, ChatStoreDependencies } from "./store.svelte"; export { createChatStore } from "./store.svelte"; export { default as ChatView } from "./ui/ChatView.svelte"; export { default as Composer } from "./ui/Composer.svelte"; export { default as ModelSelector } from "./ui/ModelSelector.svelte"; +export { default as TurnSummary } from "./ui/TurnSummary.svelte"; diff --git a/src/features/chat/store.svelte.ts b/src/features/chat/store.svelte.ts index 1d8ab17..58c165f 100644 --- a/src/features/chat/store.svelte.ts +++ b/src/features/chat/store.svelte.ts @@ -13,6 +13,8 @@ import { selectChunks, selectMessages, } from "../../core/chunks"; +import type { TelemetryState } from "../../core/telemetry"; +import { foldMetricEvent, initialState as telemetryInitialState } from "../../core/telemetry"; import type { ConversationCache } from "../conversation-cache"; import type { ChatTransport, HistorySync } from "./ports"; @@ -30,6 +32,8 @@ export interface ChatStore { readonly pendingSync: boolean; readonly error: string | null; readonly model: string | undefined; + readonly telemetry: TelemetryState; + readonly currentTurnId: string | null; handleDelta(msg: ChatDeltaMessage | ChatErrorMessage): void; send(text: string): void; setModel(model: string): void; @@ -42,6 +46,7 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore { let _pendingSync = $state(false); let _error = $state<string | null>(null); let _model = $state<string | undefined>(deps.model); + let telemetry = $state<TelemetryState>(telemetryInitialState()); let disposed = false; async function syncTail(): Promise<void> { @@ -76,6 +81,12 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore { get model(): string | undefined { return _model; }, + get telemetry(): TelemetryState { + return telemetry; + }, + get currentTurnId(): string | null { + return transcript.currentTurnId; + }, handleDelta(msg: ChatDeltaMessage | ChatErrorMessage): void { if (msg.type === "chat.error") { @@ -89,6 +100,7 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore { return; } transcript = foldEvent(transcript, msg.event); + telemetry = foldMetricEvent(telemetry, msg.event); if (transcript.sealedTurnId !== null) { void syncTail(); } diff --git a/src/features/chat/store.test.ts b/src/features/chat/store.test.ts index 71781ac..347cdd7 100644 --- a/src/features/chat/store.test.ts +++ b/src/features/chat/store.test.ts @@ -393,6 +393,52 @@ describe("createChatStore", () => { store.dispose(); }); + it("folding step-complete and usage events populates telemetry", () => { + const transport = createFakeTransport(); + const historySync = createFakeHistorySync(); + const cache = createFakeCache(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: historySync.impl, + cache: cache.impl, + }); + + store.handleDelta(deltaEvent({ type: "turn-start", conversationId: CONV_ID, turnId: "t1" })); + store.handleDelta( + deltaEvent({ + type: "step-complete", + conversationId: CONV_ID, + turnId: "t1", + stepId: "t1#0" as StepId, + ttftMs: 300, + decodeMs: 700, + genTotalMs: 1000, + }), + ); + store.handleDelta( + deltaEvent({ + type: "usage", + conversationId: CONV_ID, + turnId: "t1", + stepId: "t1#0" as StepId, + usage: { inputTokens: 50, outputTokens: 20 }, + }), + ); + + const turn = store.telemetry.turns.get("t1"); + expect(turn).toBeDefined(); + expect(turn?.steps).toHaveLength(1); + const step = turn?.steps.find((s) => s.stepId === ("t1#0" as StepId)); + expect(step).toBeDefined(); + expect(step?.ttftMs).toBe(300); + expect(step?.decodeMs).toBe(700); + expect(step?.usage?.inputTokens).toBe(50); + expect(step?.usage?.outputTokens).toBe(20); + + store.dispose(); + }); + it("handleDelta ignores a chat.delta for a different conversationId", () => { const transport = createFakeTransport(); const historySync = createFakeHistorySync(); diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts index b31cbf1..02d3c5a 100644 --- a/src/features/chat/ui.test.ts +++ b/src/features/chat/ui.test.ts @@ -3,9 +3,15 @@ import { render, screen } from "@testing-library/svelte"; import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import type { RenderedChunk } from "../../core/chunks"; +import type { TelemetryState } from "../../core/telemetry"; +import { initialState } from "../../core/telemetry"; import ChatView from "./ui/ChatView.svelte"; import Composer from "./ui/Composer.svelte"; import ModelSelector from "./ui/ModelSelector.svelte"; +import TurnSummary from "./ui/TurnSummary.svelte"; + +const emptyTelemetry = initialState(); +const noTurnId = null; describe("ChatView", () => { it("renders a message's text chunk", () => { @@ -18,7 +24,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("Hello world")).toBeInTheDocument(); }); @@ -34,7 +40,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("Hi there")).toBeInTheDocument(); expect(screen.getByText("Hello!")).toBeInTheDocument(); @@ -55,7 +61,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("read_file")).toBeInTheDocument(); const pre = screen.getByText((content, element) => { @@ -80,7 +86,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("read_file")).toBeInTheDocument(); expect(screen.getByText("file contents here")).toBeInTheDocument(); @@ -96,7 +102,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); const alert = screen.getByRole("alert"); expect(alert).toHaveTextContent("Something failed"); @@ -112,7 +118,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("Rate limited")).toBeInTheDocument(); expect(screen.getByText("[RATE_LIMIT]")).toBeInTheDocument(); @@ -128,7 +134,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("System context loaded")).toBeInTheDocument(); }); @@ -143,7 +149,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); // In-flight chunks render at full opacity (no faded "disabled" look). const wrapper = screen.getByText("Streaming...").closest("div"); @@ -151,7 +157,7 @@ describe("ChatView", () => { }); it("renders empty transcript", () => { - render(ChatView, { props: { chunks: [] } }); + render(ChatView, { props: { chunks: [], telemetry: emptyTelemetry, currentTurnId: noTurnId } }); const log = screen.getByRole("log"); expect(log).toBeInTheDocument(); @@ -199,7 +205,9 @@ describe("ChatView", () => { }, ]; - const { container } = render(ChatView, { props: { chunks } }); + const { container } = render(ChatView, { + props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId }, + }); // One DaisyUI list with two rows (one per call), not separate cards. const lists = container.querySelectorAll("ul.list"); @@ -224,7 +232,9 @@ describe("ChatView", () => { }, ]; - const { container } = render(ChatView, { props: { chunks } }); + const { container } = render(ChatView, { + props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId }, + }); const collapse = container.querySelector(".collapse"); expect(collapse).not.toBeNull(); @@ -247,7 +257,9 @@ describe("ChatView", () => { }, ]; - const { container, rerender } = render(ChatView, { props: { chunks: streaming } }); + const { container, rerender } = render(ChatView, { + props: { chunks: streaming, telemetry: emptyTelemetry, currentTurnId: noTurnId }, + }); // Streaming: "Thinking" + loading dots. expect(screen.getByText("Thinking")).toBeInTheDocument(); @@ -269,6 +281,8 @@ describe("ChatView", () => { provisional: false, }, ], + telemetry: emptyTelemetry, + currentTurnId: noTurnId, }); // Completed: "Thoughts", no dots — and the open state survived the transition. @@ -278,6 +292,118 @@ describe("ChatView", () => { expect(screen.getByRole("checkbox", { name: "Toggle thoughts" })).toBeChecked(); expect(container).toHaveTextContent("hmm, all done"); }); + + it("assistant text shows step metrics footer when step-complete data is available", () => { + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "assistant", + chunk: { type: "text", text: "Here is my answer" }, + provisional: false, + }, + ]; + + const telemetry: TelemetryState = { + turns: new Map([ + [ + "turn-1", + { + wallMs: 2500, + steps: [ + { + stepId: "turn-1#0" as StepId, + genTotalMs: 1200, + decodeMs: 1000, + usage: { inputTokens: 100, outputTokens: 86 }, + }, + ], + }, + ], + ]), + }; + + render(ChatView, { props: { chunks, telemetry, currentTurnId: "turn-1" } }); + + expect(screen.getByText("Here is my answer")).toBeInTheDocument(); + expect(screen.getByText("1.2s")).toBeInTheDocument(); + expect(screen.getByText("86 t/s")).toBeInTheDocument(); + expect(screen.getByText("86 tok")).toBeInTheDocument(); + }); + + it("does not show metrics footer when no step data exists", () => { + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "assistant", + chunk: { type: "text", text: "Still streaming" }, + provisional: true, + }, + ]; + + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: "turn-1" } }); + + expect(screen.getByText("Still streaming")).toBeInTheDocument(); + expect(screen.queryByText("t/s")).toBeNull(); + expect(screen.queryByText("tok")).toBeNull(); + }); +}); + +describe("TurnSummary", () => { + it("renders turn stats when telemetry has data", () => { + const telemetry: TelemetryState = { + turns: new Map([ + [ + "turn-1", + { + wallMs: 4200, + steps: [ + { + stepId: "turn-1#0" as StepId, + genTotalMs: 2000, + decodeMs: 1500, + usage: { inputTokens: 500, outputTokens: 300 }, + }, + { + stepId: "turn-1#1" as StepId, + genTotalMs: 1800, + decodeMs: 1200, + usage: { inputTokens: 600, outputTokens: 200 }, + }, + ], + }, + ], + ]), + }; + + render(TurnSummary, { props: { telemetry, turnId: "turn-1" } }); + + expect(screen.getByText("Turn")).toBeInTheDocument(); + expect(screen.getByText("4.2s")).toBeInTheDocument(); + expect(screen.getByText("Tokens")).toBeInTheDocument(); + expect(screen.getByText("1,600")).toBeInTheDocument(); + expect(screen.getByText("Output")).toBeInTheDocument(); + expect(screen.getByText("500")).toBeInTheDocument(); + expect(screen.getByText("Input")).toBeInTheDocument(); + expect(screen.getByText("1,100")).toBeInTheDocument(); + expect(screen.getByText("Steps")).toBeInTheDocument(); + expect(screen.getByText("2")).toBeInTheDocument(); + expect(screen.getByText("TPS")).toBeInTheDocument(); + expect(screen.getByText("185 t/s")).toBeInTheDocument(); + }); + + it("renders nothing when turnId is null", () => { + const { container } = render(TurnSummary, { + props: { telemetry: emptyTelemetry, turnId: null }, + }); + expect(container.querySelector(".stats")).toBeNull(); + }); + + it("renders nothing when turn metrics not found", () => { + const { container } = render(TurnSummary, { + props: { telemetry: emptyTelemetry, turnId: "nonexistent" }, + }); + expect(container.querySelector(".stats")).toBeNull(); + }); }); describe("Composer", () => { diff --git a/src/features/chat/ui/ChatView.svelte b/src/features/chat/ui/ChatView.svelte index 3a078fb..6acda53 100644 --- a/src/features/chat/ui/ChatView.svelte +++ b/src/features/chat/ui/ChatView.svelte @@ -1,16 +1,27 @@ <script lang="ts"> import { groupRenderedChunks, type RenderedChunk } from "../index"; + import type { TelemetryState } from "../../../core/telemetry"; + import { stepMetrics, stepTps } from "../../../core/telemetry"; - let { chunks }: { chunks: readonly RenderedChunk[] } = $props(); + interface Props { + chunks: readonly RenderedChunk[]; + telemetry: TelemetryState; + currentTurnId: string | null; + } + + let { chunks, telemetry, currentTurnId }: Props = $props(); const groups = $derived(groupRenderedChunks(chunks)); - // Stable per-row keys. Thinking blocks get an ordinal key (`think<n>`) that - // survives the provisional→committed (seq null → seq N) transition, so the - // collapse's open/close state is NOT lost when a turn seals. (App isolates - // these keys per conversation via {#key}.) + function formatMs(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms`; + const s = ms / 1000; + return s < 60 ? `${s.toFixed(1)}s` : `${Math.floor(s / 60)}m${Math.round(s % 60)}s`; + } + const rows = $derived.by(() => { let thinking = 0; + let stepIdx = 0; return groups.map((group, i) => { let key: string; if (group.kind === "tool-batch") { @@ -22,14 +33,17 @@ } else { key = `p${i}`; } - return { group, key }; + const si = stepIdx; + if (group.kind === "tool-batch" || (group.kind === "single" && (group.chunk.chunk.type === "tool-call" || group.chunk.chunk.type === "tool-result"))) { + stepIdx++; + } + return { group, key, stepIdx: si }; }); }); </script> -{#snippet chunkRow(rendered: RenderedChunk)} +{#snippet chunkRow(rendered: RenderedChunk, sIdx: number)} {#if rendered.role === "user"} - <!-- User: a speech bubble, left-aligned --> <div class="chat chat-start"> <div class="chat-bubble chat-bubble-primary"> {#if rendered.chunk.type === "text"} @@ -38,9 +52,6 @@ </div> </div> {:else if rendered.chunk.type === "thinking"} - <!-- Thinking: a visible bubble (like tool cards), holding a checkbox collapse - (no arrow icon, smooth open/close). Title reads "Thinking" + loading dots - while generating, then "Thoughts" with no dots once complete. --> <div class="chat chat-start [&>.chat-bubble]:max-w-5xl [&>.chat-bubble]:p-0"> <div class="chat-bubble w-full bg-transparent"> <div class="collapse w-full rounded-box bg-base-200 text-sm"> @@ -58,14 +69,18 @@ </div> </div> {:else if rendered.chunk.type === "tool-call" || rendered.chunk.type === "tool-result"} - <!-- Single tool call/result: a regular (non-speech) card. Nested in the - chat-start grid via a transparent, padding-stripped chat-bubble shim so - the card inherits the same left offset as the bubble bodies. --> + {@const step = currentTurnId ? stepMetrics(telemetry, currentTurnId, sIdx) : undefined} + {@const toolDur = step?.toolDurationMs} <div class="chat chat-start [&>.chat-bubble]:max-w-full [&>.chat-bubble]:p-0"> <div class="chat-bubble bg-transparent"> {#if rendered.chunk.type === "tool-call"} <div class="w-fit max-w-full rounded-box bg-base-200 p-3 text-sm"> - <strong>{rendered.chunk.toolName}</strong> + <div class="flex items-center gap-2"> + <strong>{rendered.chunk.toolName}</strong> + {#if toolDur !== undefined && toolDur > 0} + <span class="badge badge-ghost badge-xs ml-auto">{formatMs(toolDur)}</span> + {/if} + </div> <pre class="text-xs mt-1">{JSON.stringify(rendered.chunk.input, null, 2)}</pre> </div> {:else} @@ -73,19 +88,43 @@ class="w-fit max-w-full rounded-box bg-base-200 p-3 text-sm" class:text-error={rendered.chunk.isError} > - <strong>{rendered.chunk.toolName}</strong> + <div class="flex items-center gap-2"> + <strong>{rendered.chunk.toolName}</strong> + {#if toolDur !== undefined && toolDur > 0} + <span class="badge badge-ghost badge-xs ml-auto">{formatMs(toolDur)}</span> + {/if} + </div> <pre class="text-xs mt-1">{rendered.chunk.content}</pre> </div> {/if} </div> </div> {:else} - <!-- Assistant text / system / error: an INVISIBLE speech bubble — same - chat-start grid as the user bubble, so it inherits identical left spacing. --> + {@const step = currentTurnId ? stepMetrics(telemetry, currentTurnId, sIdx) : undefined} + {@const tps = step ? stepTps(step) : undefined} <div class="chat chat-start [&>.chat-bubble]:max-w-5xl"> <div class="chat-bubble w-full bg-transparent"> {#if rendered.chunk.type === "text"} - <p>{rendered.chunk.text}</p> + <ul class="list rounded-box text-sm"> + <li class="list-row"> + <p>{rendered.chunk.text}</p> + </li> + {#if step && (step.genTotalMs !== undefined || tps !== undefined || step.usage?.outputTokens !== undefined)} + <li class="list-row"> + {#if step.genTotalMs !== undefined} + <span class="badge badge-ghost badge-xs">{formatMs(step.genTotalMs)}</span> + {/if} + <span>·</span> + {#if tps !== undefined} + <span class="badge badge-ghost badge-xs">{Math.round(tps)} t/s</span> + {/if} + <span>·</span> + {#if step.usage?.outputTokens !== undefined} + <span class="badge badge-ghost badge-xs">{step.usage.outputTokens} tok</span> + {/if} + </li> + {/if} + </ul> {:else if rendered.chunk.type === "error"} <div class="text-error" role="alert"> {rendered.chunk.message} @@ -102,20 +141,24 @@ {/snippet} <div class="flex flex-col gap-2 p-4 pl-6" role="log" aria-live="polite"> - {#each rows as { group, key } (key)} + {#each rows as { group, key, stepIdx } (key)} {#if group.kind === "single"} - {@render chunkRow(group.chunk)} + {@render chunkRow(group.chunk, stepIdx)} {:else} - <!-- Batched tool calls (one step): a single bubble holding a DaisyUI list, - one row per call paired with its result. Same chat-start grid shim as - the single tool card so it lines up with the other messages. --> + {@const step = currentTurnId ? stepMetrics(telemetry, currentTurnId, stepIdx) : undefined} + {@const toolDur = step?.toolDurationMs} <div class="chat chat-start [&>.chat-bubble]:max-w-full [&>.chat-bubble]:p-0"> <div class="chat-bubble bg-transparent"> <ul class="list w-fit max-w-full rounded-box bg-base-200 text-sm"> {#each group.entries as entry (entry.call.toolCallId)} <li class="list-row"> <div> - <strong>{entry.call.toolName}</strong> + <div class="flex items-center gap-2"> + <strong>{entry.call.toolName}</strong> + {#if toolDur !== undefined && toolDur > 0} + <span class="badge badge-ghost badge-xs ml-auto">{formatMs(toolDur)}</span> + {/if} + </div> <pre class="text-xs mt-1">{JSON.stringify(entry.call.input, null, 2)}</pre> {#if entry.result} <pre diff --git a/src/features/chat/ui/TurnSummary.svelte b/src/features/chat/ui/TurnSummary.svelte new file mode 100644 index 0000000..eedb0cc --- /dev/null +++ b/src/features/chat/ui/TurnSummary.svelte @@ -0,0 +1,75 @@ +<script lang="ts"> + import type { TelemetryState } from "../../../core/telemetry"; + import { + stepCount, + totalInputTokens, + totalOutputTokens, + turnMetrics, + turnTps, + } from "../../../core/telemetry"; + + interface Props { + telemetry: TelemetryState; + turnId: string | null; + } + + let { telemetry, turnId }: Props = $props(); + + function formatMs(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms`; + const s = ms / 1000; + return s < 60 ? `${s.toFixed(1)}s` : `${Math.floor(s / 60)}m${Math.round(s % 60)}s`; + } + + const stats = $derived.by(() => { + if (turnId === null) return null; + const metrics = turnMetrics(telemetry, turnId); + if (metrics === undefined) return null; + + const items: { label: string; value: string }[] = []; + + if (metrics.wallMs !== undefined) { + items.push({ label: "Turn", value: formatMs(metrics.wallMs) }); + } + + const outTokens = totalOutputTokens(telemetry, turnId); + const inTokens = totalInputTokens(telemetry, turnId); + if (outTokens !== undefined || inTokens !== undefined) { + const total = (outTokens ?? 0) + (inTokens ?? 0); + items.push({ label: "Tokens", value: total.toLocaleString() }); + } + if (outTokens !== undefined) { + items.push({ label: "Output", value: outTokens.toLocaleString() }); + } + if (inTokens !== undefined) { + items.push({ label: "Input", value: inTokens.toLocaleString() }); + } + + const count = stepCount(telemetry, turnId); + if (count > 0) { + items.push({ label: "Steps", value: String(count) }); + } + + const tps = turnTps(telemetry, turnId); + if (tps !== undefined) { + items.push({ label: "TPS", value: `${Math.round(tps)} t/s` }); + } + + return items; + }); +</script> + +{#if stats !== null} + <div class="chat chat-start [&>.chat-bubble]:max-w-5xl"> + <div class="chat-bubble w-full bg-transparent"> + <div class="stats stats-vertical lg:stats-horizontal"> + {#each stats as stat} + <div class="stat"> + <div class="stat-title">{stat.label}</div> + <div class="stat-value text-sm">{stat.value}</div> + </div> + {/each} + </div> + </div> + </div> +{/if} |
