diff options
Diffstat (limited to 'packages/session-orchestrator/src/metrics.test.ts')
| -rw-r--r-- | packages/session-orchestrator/src/metrics.test.ts | 720 |
1 files changed, 360 insertions, 360 deletions
diff --git a/packages/session-orchestrator/src/metrics.test.ts b/packages/session-orchestrator/src/metrics.test.ts index 1920fc0..2ecaae8 100644 --- a/packages/session-orchestrator/src/metrics.test.ts +++ b/packages/session-orchestrator/src/metrics.test.ts @@ -3,367 +3,367 @@ import { describe, expect, it } from "vitest"; import { createMetricsAccumulator } from "./metrics.js"; function stepId(id: string): StepId { - return id as StepId; + return id as StepId; } describe("createMetricsAccumulator", () => { - it("builds a TurnMetrics from a single-step turn", () => { - const acc = createMetricsAccumulator(); - - const usageEvent: AgentEvent = { - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - usage: { inputTokens: 10, outputTokens: 5 }, - }; - const stepCompleteEvent: AgentEvent = { - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - ttftMs: 50, - decodeMs: 150, - genTotalMs: 200, - }; - const doneEvent: AgentEvent = { - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - durationMs: 500, - usage: { inputTokens: 10, outputTokens: 5 }, - }; - - acc.ingest(usageEvent); - acc.ingest(stepCompleteEvent); - acc.ingest(doneEvent); - - const tm = acc.build("t1"); - expect(tm.turnId).toBe("t1"); - expect(tm.usage.inputTokens).toBe(10); - expect(tm.usage.outputTokens).toBe(5); - expect(tm.durationMs).toBe(500); - expect(tm.steps).toHaveLength(1); - expect(tm.steps[0]?.stepId).toBe(stepId("t1#0")); - expect(tm.steps[0]?.usage.inputTokens).toBe(10); - expect(tm.steps[0]?.usage.outputTokens).toBe(5); - expect(tm.steps[0]?.ttftMs).toBe(50); - expect(tm.steps[0]?.decodeMs).toBe(150); - expect(tm.steps[0]?.genTotalMs).toBe(200); - }); - - it("aggregates multi-step usage and preserves step order", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - usage: { inputTokens: 10, outputTokens: 5 }, - }); - acc.ingest({ - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - genTotalMs: 100, - }); - acc.ingest({ - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#1"), - usage: { inputTokens: 20, outputTokens: 10 }, - }); - acc.ingest({ - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#1"), - genTotalMs: 150, - }); - acc.ingest({ - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - usage: { inputTokens: 30, outputTokens: 15 }, - }); - - const tm = acc.build("t1"); - expect(tm.steps).toHaveLength(2); - expect(tm.steps[0]?.stepId).toBe(stepId("t1#0")); - expect(tm.steps[0]?.usage.inputTokens).toBe(10); - expect(tm.steps[1]?.stepId).toBe(stepId("t1#1")); - expect(tm.steps[1]?.usage.inputTokens).toBe(20); - expect(tm.usage.inputTokens).toBe(30); - expect(tm.usage.outputTokens).toBe(15); - }); - - it("joins per-step timing and usage by stepId", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - ttftMs: 50, - decodeMs: 100, - genTotalMs: 150, - }); - acc.ingest({ - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - const tm = acc.build("t1"); - expect(tm.steps).toHaveLength(1); - expect(tm.steps[0]?.usage.inputTokens).toBe(10); - expect(tm.steps[0]?.ttftMs).toBe(50); - expect(tm.steps[0]?.decodeMs).toBe(100); - expect(tm.steps[0]?.genTotalMs).toBe(150); - }); - - it("turn-level usage comes from done event, not step sum", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - usage: { inputTokens: 10, outputTokens: 5 }, - }); - acc.ingest({ - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - }); - acc.ingest({ - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - usage: { inputTokens: 100, outputTokens: 50 }, - }); - - const tm = acc.build("t1"); - expect(tm.usage.inputTokens).toBe(100); - expect(tm.usage.outputTokens).toBe(50); - }); - - it("falls back to summing step usage when done.usage is absent", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - usage: { inputTokens: 10, outputTokens: 5 }, - }); - acc.ingest({ - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - }); - acc.ingest({ - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - }); - - const tm = acc.build("t1"); - expect(tm.usage.inputTokens).toBe(10); - expect(tm.usage.outputTokens).toBe(5); - }); - - it("tolerates missing usage on a step (timing only)", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - genTotalMs: 200, - }); - acc.ingest({ - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - }); - - const tm = acc.build("t1"); - expect(tm.steps).toHaveLength(1); - expect(tm.steps[0]?.usage.inputTokens).toBe(0); - expect(tm.steps[0]?.usage.outputTokens).toBe(0); - expect(tm.steps[0]?.genTotalMs).toBe(200); - }); - - it("tolerates missing timing on a step (usage only)", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - usage: { inputTokens: 10, outputTokens: 5 }, - }); - acc.ingest({ - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - }); - - const tm = acc.build("t1"); - expect(tm.steps).toHaveLength(1); - expect(tm.steps[0]?.usage.inputTokens).toBe(10); - expect(tm.steps[0]?.ttftMs).toBeUndefined(); - expect(tm.steps[0]?.decodeMs).toBeUndefined(); - expect(tm.steps[0]?.genTotalMs).toBeUndefined(); - }); - - it("reset clears all accumulated state", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - usage: { inputTokens: 10, outputTokens: 5 }, - }); - acc.ingest({ - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - acc.reset(); - - const tm = acc.build("t2"); - expect(tm.steps).toHaveLength(0); - expect(tm.usage.inputTokens).toBe(0); - expect(tm.usage.outputTokens).toBe(0); - }); - - it("contextSize equals inputTokens + outputTokens for a single-step turn", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - usage: { inputTokens: 10, outputTokens: 5 }, - }); - acc.ingest({ - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - }); - acc.ingest({ - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - usage: { inputTokens: 10, outputTokens: 5 }, - }); - - const tm = acc.build("t1"); - expect(tm.contextSize).toBe(15); - }); - - it("contextSize equals ONLY the last step's inputTokens + outputTokens for a multi-step turn", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - usage: { inputTokens: 10, outputTokens: 5 }, - }); - acc.ingest({ - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - }); - acc.ingest({ - type: "usage", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#1"), - usage: { inputTokens: 20, outputTokens: 10 }, - }); - acc.ingest({ - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#1"), - }); - acc.ingest({ - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - usage: { inputTokens: 100, outputTokens: 50 }, - }); - - const tm = acc.build("t1"); - expect(tm.contextSize).toBe(30); - expect(tm.contextSize).not.toBe(tm.usage.inputTokens); - }); - - it("contextSize is undefined when the turn has no steps", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - }); - - const tm = acc.build("t1"); - expect(tm.contextSize).toBeUndefined(); - }); - - it("contextSize is undefined when the last step has no usable per-step usage", () => { - const acc = createMetricsAccumulator(); - - acc.ingest({ - type: "step-complete", - conversationId: "c1", - turnId: "t1", - stepId: stepId("t1#0"), - genTotalMs: 200, - }); - acc.ingest({ - type: "done", - conversationId: "c1", - turnId: "t1", - reason: "stop", - }); - - const tm = acc.build("t1"); - expect(tm.contextSize).toBeUndefined(); - }); + it("builds a TurnMetrics from a single-step turn", () => { + const acc = createMetricsAccumulator(); + + const usageEvent: AgentEvent = { + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + usage: { inputTokens: 10, outputTokens: 5 }, + }; + const stepCompleteEvent: AgentEvent = { + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + ttftMs: 50, + decodeMs: 150, + genTotalMs: 200, + }; + const doneEvent: AgentEvent = { + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + durationMs: 500, + usage: { inputTokens: 10, outputTokens: 5 }, + }; + + acc.ingest(usageEvent); + acc.ingest(stepCompleteEvent); + acc.ingest(doneEvent); + + const tm = acc.build("t1"); + expect(tm.turnId).toBe("t1"); + expect(tm.usage.inputTokens).toBe(10); + expect(tm.usage.outputTokens).toBe(5); + expect(tm.durationMs).toBe(500); + expect(tm.steps).toHaveLength(1); + expect(tm.steps[0]?.stepId).toBe(stepId("t1#0")); + expect(tm.steps[0]?.usage.inputTokens).toBe(10); + expect(tm.steps[0]?.usage.outputTokens).toBe(5); + expect(tm.steps[0]?.ttftMs).toBe(50); + expect(tm.steps[0]?.decodeMs).toBe(150); + expect(tm.steps[0]?.genTotalMs).toBe(200); + }); + + it("aggregates multi-step usage and preserves step order", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + usage: { inputTokens: 10, outputTokens: 5 }, + }); + acc.ingest({ + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + genTotalMs: 100, + }); + acc.ingest({ + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#1"), + usage: { inputTokens: 20, outputTokens: 10 }, + }); + acc.ingest({ + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#1"), + genTotalMs: 150, + }); + acc.ingest({ + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + usage: { inputTokens: 30, outputTokens: 15 }, + }); + + const tm = acc.build("t1"); + expect(tm.steps).toHaveLength(2); + expect(tm.steps[0]?.stepId).toBe(stepId("t1#0")); + expect(tm.steps[0]?.usage.inputTokens).toBe(10); + expect(tm.steps[1]?.stepId).toBe(stepId("t1#1")); + expect(tm.steps[1]?.usage.inputTokens).toBe(20); + expect(tm.usage.inputTokens).toBe(30); + expect(tm.usage.outputTokens).toBe(15); + }); + + it("joins per-step timing and usage by stepId", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + ttftMs: 50, + decodeMs: 100, + genTotalMs: 150, + }); + acc.ingest({ + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + usage: { inputTokens: 10, outputTokens: 5 }, + }); + + const tm = acc.build("t1"); + expect(tm.steps).toHaveLength(1); + expect(tm.steps[0]?.usage.inputTokens).toBe(10); + expect(tm.steps[0]?.ttftMs).toBe(50); + expect(tm.steps[0]?.decodeMs).toBe(100); + expect(tm.steps[0]?.genTotalMs).toBe(150); + }); + + it("turn-level usage comes from done event, not step sum", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + usage: { inputTokens: 10, outputTokens: 5 }, + }); + acc.ingest({ + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + }); + acc.ingest({ + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + usage: { inputTokens: 100, outputTokens: 50 }, + }); + + const tm = acc.build("t1"); + expect(tm.usage.inputTokens).toBe(100); + expect(tm.usage.outputTokens).toBe(50); + }); + + it("falls back to summing step usage when done.usage is absent", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + usage: { inputTokens: 10, outputTokens: 5 }, + }); + acc.ingest({ + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + }); + acc.ingest({ + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + }); + + const tm = acc.build("t1"); + expect(tm.usage.inputTokens).toBe(10); + expect(tm.usage.outputTokens).toBe(5); + }); + + it("tolerates missing usage on a step (timing only)", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + genTotalMs: 200, + }); + acc.ingest({ + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + }); + + const tm = acc.build("t1"); + expect(tm.steps).toHaveLength(1); + expect(tm.steps[0]?.usage.inputTokens).toBe(0); + expect(tm.steps[0]?.usage.outputTokens).toBe(0); + expect(tm.steps[0]?.genTotalMs).toBe(200); + }); + + it("tolerates missing timing on a step (usage only)", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + usage: { inputTokens: 10, outputTokens: 5 }, + }); + acc.ingest({ + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + }); + + const tm = acc.build("t1"); + expect(tm.steps).toHaveLength(1); + expect(tm.steps[0]?.usage.inputTokens).toBe(10); + expect(tm.steps[0]?.ttftMs).toBeUndefined(); + expect(tm.steps[0]?.decodeMs).toBeUndefined(); + expect(tm.steps[0]?.genTotalMs).toBeUndefined(); + }); + + it("reset clears all accumulated state", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + usage: { inputTokens: 10, outputTokens: 5 }, + }); + acc.ingest({ + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + usage: { inputTokens: 10, outputTokens: 5 }, + }); + + acc.reset(); + + const tm = acc.build("t2"); + expect(tm.steps).toHaveLength(0); + expect(tm.usage.inputTokens).toBe(0); + expect(tm.usage.outputTokens).toBe(0); + }); + + it("contextSize equals inputTokens + outputTokens for a single-step turn", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + usage: { inputTokens: 10, outputTokens: 5 }, + }); + acc.ingest({ + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + }); + acc.ingest({ + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + usage: { inputTokens: 10, outputTokens: 5 }, + }); + + const tm = acc.build("t1"); + expect(tm.contextSize).toBe(15); + }); + + it("contextSize equals ONLY the last step's inputTokens + outputTokens for a multi-step turn", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + usage: { inputTokens: 10, outputTokens: 5 }, + }); + acc.ingest({ + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + }); + acc.ingest({ + type: "usage", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#1"), + usage: { inputTokens: 20, outputTokens: 10 }, + }); + acc.ingest({ + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#1"), + }); + acc.ingest({ + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + usage: { inputTokens: 100, outputTokens: 50 }, + }); + + const tm = acc.build("t1"); + expect(tm.contextSize).toBe(30); + expect(tm.contextSize).not.toBe(tm.usage.inputTokens); + }); + + it("contextSize is undefined when the turn has no steps", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + }); + + const tm = acc.build("t1"); + expect(tm.contextSize).toBeUndefined(); + }); + + it("contextSize is undefined when the last step has no usable per-step usage", () => { + const acc = createMetricsAccumulator(); + + acc.ingest({ + type: "step-complete", + conversationId: "c1", + turnId: "t1", + stepId: stepId("t1#0"), + genTotalMs: 200, + }); + acc.ingest({ + type: "done", + conversationId: "c1", + turnId: "t1", + reason: "stop", + }); + + const tm = acc.build("t1"); + expect(tm.contextSize).toBeUndefined(); + }); }); |