From 48c6d85c3cc5a57a729f14068e2346b17ed62088 Mon Sep 17 00:00:00 2001 From: Adam Malczewski Date: Sun, 7 Jun 2026 18:41:27 +0900 Subject: feat(chat): live turn metrics — telemetry reducer + rendering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consume wire/transport-contract 0.3.0 (step-complete event + timing fields on usage/tool-result/done). Pure core/telemetry module: foldMetricEvent (reducer) + derived selectors (stepTps, turnTps, etc). TelemetryState is pure data, no active-turn tracking — consumers pass turnId to selectors. ChatStore wires foldMetricEvent into handleDelta and exposes telemetry + currentTurnId. ChatView shows step-metrics footer (time/TPS/tokens) on assistant text bubbles and durationMs badge on tool cards. New TurnSummary component renders turn-level stats (wall-clock, tokens, steps, TPS) in a DaisyUI stats block. Extended live-probe to verify telemetry events against bin/up (pending backend restart). 336 tests, typecheck 0, biome clean, build ok. --- src/features/chat/index.ts | 2 + src/features/chat/store.svelte.ts | 12 +++ src/features/chat/store.test.ts | 46 ++++++++++ src/features/chat/ui.test.ts | 150 +++++++++++++++++++++++++++++--- src/features/chat/ui/ChatView.svelte | 93 ++++++++++++++------ src/features/chat/ui/TurnSummary.svelte | 75 ++++++++++++++++ 6 files changed, 341 insertions(+), 37 deletions(-) create mode 100644 src/features/chat/ui/TurnSummary.svelte (limited to 'src/features') diff --git a/src/features/chat/index.ts b/src/features/chat/index.ts index 4f2091a..b096cca 100644 --- a/src/features/chat/index.ts +++ b/src/features/chat/index.ts @@ -1,8 +1,10 @@ export type { RenderedChunk, RenderGroup, ToolBatchEntry } from "../../core/chunks"; export { groupRenderedChunks } from "../../core/chunks"; +export type { StepMetrics, TelemetryState, TurnMetrics } from "../../core/telemetry"; export type { ChatTransport, HistorySync } from "./ports"; export type { ChatStore, ChatStoreDependencies } from "./store.svelte"; export { createChatStore } from "./store.svelte"; export { default as ChatView } from "./ui/ChatView.svelte"; export { default as Composer } from "./ui/Composer.svelte"; export { default as ModelSelector } from "./ui/ModelSelector.svelte"; +export { default as TurnSummary } from "./ui/TurnSummary.svelte"; diff --git a/src/features/chat/store.svelte.ts b/src/features/chat/store.svelte.ts index 1d8ab17..58c165f 100644 --- a/src/features/chat/store.svelte.ts +++ b/src/features/chat/store.svelte.ts @@ -13,6 +13,8 @@ import { selectChunks, selectMessages, } from "../../core/chunks"; +import type { TelemetryState } from "../../core/telemetry"; +import { foldMetricEvent, initialState as telemetryInitialState } from "../../core/telemetry"; import type { ConversationCache } from "../conversation-cache"; import type { ChatTransport, HistorySync } from "./ports"; @@ -30,6 +32,8 @@ export interface ChatStore { readonly pendingSync: boolean; readonly error: string | null; readonly model: string | undefined; + readonly telemetry: TelemetryState; + readonly currentTurnId: string | null; handleDelta(msg: ChatDeltaMessage | ChatErrorMessage): void; send(text: string): void; setModel(model: string): void; @@ -42,6 +46,7 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore { let _pendingSync = $state(false); let _error = $state(null); let _model = $state(deps.model); + let telemetry = $state(telemetryInitialState()); let disposed = false; async function syncTail(): Promise { @@ -76,6 +81,12 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore { get model(): string | undefined { return _model; }, + get telemetry(): TelemetryState { + return telemetry; + }, + get currentTurnId(): string | null { + return transcript.currentTurnId; + }, handleDelta(msg: ChatDeltaMessage | ChatErrorMessage): void { if (msg.type === "chat.error") { @@ -89,6 +100,7 @@ export function createChatStore(deps: ChatStoreDependencies): ChatStore { return; } transcript = foldEvent(transcript, msg.event); + telemetry = foldMetricEvent(telemetry, msg.event); if (transcript.sealedTurnId !== null) { void syncTail(); } diff --git a/src/features/chat/store.test.ts b/src/features/chat/store.test.ts index 71781ac..347cdd7 100644 --- a/src/features/chat/store.test.ts +++ b/src/features/chat/store.test.ts @@ -393,6 +393,52 @@ describe("createChatStore", () => { store.dispose(); }); + it("folding step-complete and usage events populates telemetry", () => { + const transport = createFakeTransport(); + const historySync = createFakeHistorySync(); + const cache = createFakeCache(); + const store = createChatStore({ + conversationId: CONV_ID, + transport: transport.impl, + historySync: historySync.impl, + cache: cache.impl, + }); + + store.handleDelta(deltaEvent({ type: "turn-start", conversationId: CONV_ID, turnId: "t1" })); + store.handleDelta( + deltaEvent({ + type: "step-complete", + conversationId: CONV_ID, + turnId: "t1", + stepId: "t1#0" as StepId, + ttftMs: 300, + decodeMs: 700, + genTotalMs: 1000, + }), + ); + store.handleDelta( + deltaEvent({ + type: "usage", + conversationId: CONV_ID, + turnId: "t1", + stepId: "t1#0" as StepId, + usage: { inputTokens: 50, outputTokens: 20 }, + }), + ); + + const turn = store.telemetry.turns.get("t1"); + expect(turn).toBeDefined(); + expect(turn?.steps).toHaveLength(1); + const step = turn?.steps.find((s) => s.stepId === ("t1#0" as StepId)); + expect(step).toBeDefined(); + expect(step?.ttftMs).toBe(300); + expect(step?.decodeMs).toBe(700); + expect(step?.usage?.inputTokens).toBe(50); + expect(step?.usage?.outputTokens).toBe(20); + + store.dispose(); + }); + it("handleDelta ignores a chat.delta for a different conversationId", () => { const transport = createFakeTransport(); const historySync = createFakeHistorySync(); diff --git a/src/features/chat/ui.test.ts b/src/features/chat/ui.test.ts index b31cbf1..02d3c5a 100644 --- a/src/features/chat/ui.test.ts +++ b/src/features/chat/ui.test.ts @@ -3,9 +3,15 @@ import { render, screen } from "@testing-library/svelte"; import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import type { RenderedChunk } from "../../core/chunks"; +import type { TelemetryState } from "../../core/telemetry"; +import { initialState } from "../../core/telemetry"; import ChatView from "./ui/ChatView.svelte"; import Composer from "./ui/Composer.svelte"; import ModelSelector from "./ui/ModelSelector.svelte"; +import TurnSummary from "./ui/TurnSummary.svelte"; + +const emptyTelemetry = initialState(); +const noTurnId = null; describe("ChatView", () => { it("renders a message's text chunk", () => { @@ -18,7 +24,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("Hello world")).toBeInTheDocument(); }); @@ -34,7 +40,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("Hi there")).toBeInTheDocument(); expect(screen.getByText("Hello!")).toBeInTheDocument(); @@ -55,7 +61,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("read_file")).toBeInTheDocument(); const pre = screen.getByText((content, element) => { @@ -80,7 +86,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("read_file")).toBeInTheDocument(); expect(screen.getByText("file contents here")).toBeInTheDocument(); @@ -96,7 +102,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); const alert = screen.getByRole("alert"); expect(alert).toHaveTextContent("Something failed"); @@ -112,7 +118,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("Rate limited")).toBeInTheDocument(); expect(screen.getByText("[RATE_LIMIT]")).toBeInTheDocument(); @@ -128,7 +134,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); expect(screen.getByText("System context loaded")).toBeInTheDocument(); }); @@ -143,7 +149,7 @@ describe("ChatView", () => { }, ]; - render(ChatView, { props: { chunks } }); + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId } }); // In-flight chunks render at full opacity (no faded "disabled" look). const wrapper = screen.getByText("Streaming...").closest("div"); @@ -151,7 +157,7 @@ describe("ChatView", () => { }); it("renders empty transcript", () => { - render(ChatView, { props: { chunks: [] } }); + render(ChatView, { props: { chunks: [], telemetry: emptyTelemetry, currentTurnId: noTurnId } }); const log = screen.getByRole("log"); expect(log).toBeInTheDocument(); @@ -199,7 +205,9 @@ describe("ChatView", () => { }, ]; - const { container } = render(ChatView, { props: { chunks } }); + const { container } = render(ChatView, { + props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId }, + }); // One DaisyUI list with two rows (one per call), not separate cards. const lists = container.querySelectorAll("ul.list"); @@ -224,7 +232,9 @@ describe("ChatView", () => { }, ]; - const { container } = render(ChatView, { props: { chunks } }); + const { container } = render(ChatView, { + props: { chunks, telemetry: emptyTelemetry, currentTurnId: noTurnId }, + }); const collapse = container.querySelector(".collapse"); expect(collapse).not.toBeNull(); @@ -247,7 +257,9 @@ describe("ChatView", () => { }, ]; - const { container, rerender } = render(ChatView, { props: { chunks: streaming } }); + const { container, rerender } = render(ChatView, { + props: { chunks: streaming, telemetry: emptyTelemetry, currentTurnId: noTurnId }, + }); // Streaming: "Thinking" + loading dots. expect(screen.getByText("Thinking")).toBeInTheDocument(); @@ -269,6 +281,8 @@ describe("ChatView", () => { provisional: false, }, ], + telemetry: emptyTelemetry, + currentTurnId: noTurnId, }); // Completed: "Thoughts", no dots — and the open state survived the transition. @@ -278,6 +292,118 @@ describe("ChatView", () => { expect(screen.getByRole("checkbox", { name: "Toggle thoughts" })).toBeChecked(); expect(container).toHaveTextContent("hmm, all done"); }); + + it("assistant text shows step metrics footer when step-complete data is available", () => { + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "assistant", + chunk: { type: "text", text: "Here is my answer" }, + provisional: false, + }, + ]; + + const telemetry: TelemetryState = { + turns: new Map([ + [ + "turn-1", + { + wallMs: 2500, + steps: [ + { + stepId: "turn-1#0" as StepId, + genTotalMs: 1200, + decodeMs: 1000, + usage: { inputTokens: 100, outputTokens: 86 }, + }, + ], + }, + ], + ]), + }; + + render(ChatView, { props: { chunks, telemetry, currentTurnId: "turn-1" } }); + + expect(screen.getByText("Here is my answer")).toBeInTheDocument(); + expect(screen.getByText("1.2s")).toBeInTheDocument(); + expect(screen.getByText("86 t/s")).toBeInTheDocument(); + expect(screen.getByText("86 tok")).toBeInTheDocument(); + }); + + it("does not show metrics footer when no step data exists", () => { + const chunks: RenderedChunk[] = [ + { + seq: 1, + role: "assistant", + chunk: { type: "text", text: "Still streaming" }, + provisional: true, + }, + ]; + + render(ChatView, { props: { chunks, telemetry: emptyTelemetry, currentTurnId: "turn-1" } }); + + expect(screen.getByText("Still streaming")).toBeInTheDocument(); + expect(screen.queryByText("t/s")).toBeNull(); + expect(screen.queryByText("tok")).toBeNull(); + }); +}); + +describe("TurnSummary", () => { + it("renders turn stats when telemetry has data", () => { + const telemetry: TelemetryState = { + turns: new Map([ + [ + "turn-1", + { + wallMs: 4200, + steps: [ + { + stepId: "turn-1#0" as StepId, + genTotalMs: 2000, + decodeMs: 1500, + usage: { inputTokens: 500, outputTokens: 300 }, + }, + { + stepId: "turn-1#1" as StepId, + genTotalMs: 1800, + decodeMs: 1200, + usage: { inputTokens: 600, outputTokens: 200 }, + }, + ], + }, + ], + ]), + }; + + render(TurnSummary, { props: { telemetry, turnId: "turn-1" } }); + + expect(screen.getByText("Turn")).toBeInTheDocument(); + expect(screen.getByText("4.2s")).toBeInTheDocument(); + expect(screen.getByText("Tokens")).toBeInTheDocument(); + expect(screen.getByText("1,600")).toBeInTheDocument(); + expect(screen.getByText("Output")).toBeInTheDocument(); + expect(screen.getByText("500")).toBeInTheDocument(); + expect(screen.getByText("Input")).toBeInTheDocument(); + expect(screen.getByText("1,100")).toBeInTheDocument(); + expect(screen.getByText("Steps")).toBeInTheDocument(); + expect(screen.getByText("2")).toBeInTheDocument(); + expect(screen.getByText("TPS")).toBeInTheDocument(); + expect(screen.getByText("185 t/s")).toBeInTheDocument(); + }); + + it("renders nothing when turnId is null", () => { + const { container } = render(TurnSummary, { + props: { telemetry: emptyTelemetry, turnId: null }, + }); + expect(container.querySelector(".stats")).toBeNull(); + }); + + it("renders nothing when turn metrics not found", () => { + const { container } = render(TurnSummary, { + props: { telemetry: emptyTelemetry, turnId: "nonexistent" }, + }); + expect(container.querySelector(".stats")).toBeNull(); + }); }); describe("Composer", () => { diff --git a/src/features/chat/ui/ChatView.svelte b/src/features/chat/ui/ChatView.svelte index 3a078fb..6acda53 100644 --- a/src/features/chat/ui/ChatView.svelte +++ b/src/features/chat/ui/ChatView.svelte @@ -1,16 +1,27 @@ -{#snippet chunkRow(rendered: RenderedChunk)} +{#snippet chunkRow(rendered: RenderedChunk, sIdx: number)} {#if rendered.role === "user"} -
{#if rendered.chunk.type === "text"} @@ -38,9 +52,6 @@
{:else if rendered.chunk.type === "thinking"} -
@@ -58,14 +69,18 @@
{:else if rendered.chunk.type === "tool-call" || rendered.chunk.type === "tool-result"} - + {@const step = currentTurnId ? stepMetrics(telemetry, currentTurnId, sIdx) : undefined} + {@const toolDur = step?.toolDurationMs}
{#if rendered.chunk.type === "tool-call"}
- {rendered.chunk.toolName} +
+ {rendered.chunk.toolName} + {#if toolDur !== undefined && toolDur > 0} + {formatMs(toolDur)} + {/if} +
{JSON.stringify(rendered.chunk.input, null, 2)}
{:else} @@ -73,19 +88,43 @@ class="w-fit max-w-full rounded-box bg-base-200 p-3 text-sm" class:text-error={rendered.chunk.isError} > - {rendered.chunk.toolName} +
+ {rendered.chunk.toolName} + {#if toolDur !== undefined && toolDur > 0} + {formatMs(toolDur)} + {/if} +
{rendered.chunk.content}
{/if}
{:else} - + {@const step = currentTurnId ? stepMetrics(telemetry, currentTurnId, sIdx) : undefined} + {@const tps = step ? stepTps(step) : undefined}
{#if rendered.chunk.type === "text"} -

{rendered.chunk.text}

+
    +
  • +

    {rendered.chunk.text}

    +
  • + {#if step && (step.genTotalMs !== undefined || tps !== undefined || step.usage?.outputTokens !== undefined)} +
  • + {#if step.genTotalMs !== undefined} + {formatMs(step.genTotalMs)} + {/if} + · + {#if tps !== undefined} + {Math.round(tps)} t/s + {/if} + · + {#if step.usage?.outputTokens !== undefined} + {step.usage.outputTokens} tok + {/if} +
  • + {/if} +
{:else if rendered.chunk.type === "error"}