diff options
Diffstat (limited to 'src/core/telemetry/reducer.test.ts')
| -rw-r--r-- | src/core/telemetry/reducer.test.ts | 252 |
1 files changed, 252 insertions, 0 deletions
diff --git a/src/core/telemetry/reducer.test.ts b/src/core/telemetry/reducer.test.ts new file mode 100644 index 0000000..119bf96 --- /dev/null +++ b/src/core/telemetry/reducer.test.ts @@ -0,0 +1,252 @@ +import type { StepId, Usage } from "@dispatch/wire"; +import { describe, expect, it } from "vitest"; +import { foldMetricEvent, initialState } from "./reducer"; +import { + stepCount, + stepMetrics, + stepToolDuration, + stepTps, + totalDecodeMs, + totalInputTokens, + totalOutputTokens, + turnMetrics, + turnTps, + turnTtft, +} from "./selectors"; + +const sid = (s: string) => s as StepId; + +const usage = (turnId: string, stepId: string, u: Usage) => ({ + type: "usage" as const, + conversationId: "c1", + turnId, + stepId: sid(stepId), + usage: u, +}); + +const stepComplete = ( + turnId: string, + stepId: string, + timing: { ttftMs?: number; decodeMs?: number; genTotalMs?: number }, +) => ({ + type: "step-complete" as const, + conversationId: "c1", + turnId, + stepId: sid(stepId), + ...timing, +}); + +describe("foldMetricEvent", () => { + it("turn-start initializes an empty turn", () => { + const s = foldMetricEvent(initialState(), { + type: "turn-start", + conversationId: "c1", + turnId: "t1", + }); + expect(s.turns.get("t1")?.steps).toEqual([]); + }); + + it("step-complete populates timing on a new step", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent( + s, + stepComplete("t1", "s0", { ttftMs: 300, decodeMs: 800, genTotalMs: 1100 }), + ); + + const step = stepMetrics(s, "t1", 0); + expect(step?.ttftMs).toBe(300); + expect(step?.decodeMs).toBe(800); + expect(step?.genTotalMs).toBe(1100); + }); + + it("usage merges tokens into a step (joined by stepId)", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, stepComplete("t1", "s0", { genTotalMs: 500 })); + s = foldMetricEvent(s, usage("t1", "s0", { inputTokens: 100, outputTokens: 50 })); + + const step = stepMetrics(s, "t1", 0); + expect(step?.usage?.inputTokens).toBe(100); + expect(step?.usage?.outputTokens).toBe(50); + expect(step?.genTotalMs).toBe(500); // timing preserved + }); + + it("usage without stepId is ignored", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, { + type: "usage", + conversationId: "c1", + turnId: "t1", + usage: { inputTokens: 100, outputTokens: 50 }, + // no stepId + }); + expect(s.turns.get("t1")?.steps).toEqual([]); + }); + + it("tool-result accumulates durationMs into its step", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, stepComplete("t1", "s0", {})); + s = foldMetricEvent(s, { + type: "tool-result", + conversationId: "c1", + turnId: "t1", + stepId: sid("s0"), + toolCallId: "tc1", + toolName: "bash", + content: "", + isError: false, + durationMs: 120, + }); + s = foldMetricEvent(s, { + type: "tool-result", + conversationId: "c1", + turnId: "t1", + stepId: sid("s0"), + toolCallId: "tc2", + toolName: "bash", + content: "", + isError: false, + durationMs: 80, + }); + + const step = stepMetrics(s, "t1", 0); + expect(step?.toolDurationMs).toBe(200); + }); + + it("done records turn wall-clock and aggregate usage", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, { + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "complete", + durationMs: 4200, + usage: { inputTokens: 800, outputTokens: 200 }, + }); + + const turn = turnMetrics(s, "t1"); + expect(turn?.wallMs).toBe(4200); + expect(turn?.doneUsage?.outputTokens).toBe(200); + }); + + it("events for an unknown turn are handled gracefully (step-complete, usage)", () => { + const s = initialState(); + // step-complete for a turn we haven't started — creates the turn. + const s2 = foldMetricEvent(s, stepComplete("t1", "s0", { ttftMs: 100 })); + expect(s2.turns.get("t1")?.steps[0]?.ttftMs).toBe(100); + }); + + it("multiple steps accumulate in order", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, stepComplete("t1", "s0", { genTotalMs: 100 })); + s = foldMetricEvent(s, stepComplete("t1", "s1", { genTotalMs: 200 })); + + expect(stepCount(s, "t1")).toBe(2); + expect(stepMetrics(s, "t1", 0)?.genTotalMs).toBe(100); + expect(stepMetrics(s, "t1", 1)?.genTotalMs).toBe(200); + }); + + it("non-metric events are no-ops", () => { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent(s, { + type: "text-delta", + conversationId: "c1", + turnId: "t1", + delta: "hi", + }); + s = foldMetricEvent(s, { + type: "turn-sealed", + conversationId: "c1", + turnId: "t1", + }); + expect(s.turns.get("t1")?.steps).toEqual([]); + }); +}); + +describe("selectors — derived metrics", () => { + function populatedState() { + let s = initialState(); + s = foldMetricEvent(s, { type: "turn-start", conversationId: "c1", turnId: "t1" }); + s = foldMetricEvent( + s, + stepComplete("t1", "s0", { ttftMs: 300, decodeMs: 700, genTotalMs: 1000 }), + ); + s = foldMetricEvent(s, usage("t1", "s0", { inputTokens: 500, outputTokens: 100 })); + s = foldMetricEvent( + s, + stepComplete("t1", "s1", { ttftMs: 200, decodeMs: 500, genTotalMs: 700 }), + ); + s = foldMetricEvent(s, usage("t1", "s1", { inputTokens: 600, outputTokens: 80 })); + s = foldMetricEvent(s, { + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "complete", + durationMs: 3500, + usage: { inputTokens: 1100, outputTokens: 180 }, + }); + return s; + } + + it("stepTps = outputTokens / (decodeMs / 1000)", () => { + const s = populatedState(); + const step = stepMetrics(s, "t1", 0)!; + expect(stepTps(step)).toBeCloseTo(100 / 0.7, 2); + }); + + it("turnTtft returns first step's ttftMs", () => { + expect(turnTtft(populatedState(), "t1")).toBe(300); + }); + + it("totalDecodeMs sums all steps' decodeMs", () => { + expect(totalDecodeMs(populatedState(), "t1")).toBe(1200); + }); + + it("turnTps = outputTokens / (totalDecodeMs / 1000)", () => { + const s = populatedState(); + expect(turnTps(s, "t1")).toBeCloseTo(180 / 1.2, 2); + }); + + it("totalOutputTokens prefers done.usage over step sum", () => { + const s = populatedState(); + expect(totalOutputTokens(s, "t1")).toBe(180); // from done.usage + }); + + it("totalInputTokens prefers done.usage over step sum", () => { + const s = populatedState(); + expect(totalInputTokens(s, "t1")).toBe(1100); + }); + + it("stepToolDuration returns sum only when > 0", () => { + const withTools = foldMetricEvent( + foldMetricEvent(initialState(), { type: "turn-start", conversationId: "c1", turnId: "t1" }), + { + type: "tool-result", + conversationId: "c1", + turnId: "t1", + stepId: sid("s0"), + toolCallId: "tc1", + toolName: "bash", + content: "", + isError: false, + durationMs: 50, + }, + ); + const step = stepMetrics(withTools, "t1", 0)!; + expect(stepToolDuration(step)).toBe(50); + expect(stepToolDuration({ stepId: sid("s0") })).toBeUndefined(); + }); + + it("returns undefined for absent fields gracefully", () => { + const s = initialState(); + expect(turnMetrics(s, "missing")).toBeUndefined(); + expect(turnTtft(s, "missing")).toBeUndefined(); + expect(turnTps(s, "missing")).toBeUndefined(); + }); +}); |