diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/live-probe.ts | 77 |
1 files changed, 75 insertions, 2 deletions
diff --git a/scripts/live-probe.ts b/scripts/live-probe.ts index 2c4dfb9..f38c907 100644 --- a/scripts/live-probe.ts +++ b/scripts/live-probe.ts @@ -43,6 +43,13 @@ import { selectMessages, type TranscriptState, } from "../src/core/chunks/index.ts"; +import { + foldMetricEvent, + stepMetrics, + type TelemetryState, + initialState as telemetryInitialState, + turnMetrics, +} from "../src/core/telemetry/index.ts"; import { createConversationCache } from "../src/features/conversation-cache/index.ts"; const WS_URL = process.env.PROBE_WS ?? "ws://localhost:24205"; @@ -87,8 +94,15 @@ async function runTurn( socket: Socket, conversationId: string, prompt: string, -): Promise<{ state: TranscriptState; deltas: number; sealed: boolean; error: string | null }> { +): Promise<{ + state: TranscriptState; + telemetry: TelemetryState; + deltas: number; + sealed: boolean; + error: string | null; +}> { let state = initialState(); + let telemetry = telemetryInitialState(); let deltas = 0; let sealed = false; let error: string | null = null; @@ -102,6 +116,7 @@ async function runTurn( } deltas++; state = foldEvent(state, msg.event); + telemetry = foldMetricEvent(telemetry, msg.event); if (msg.event.type === "turn-sealed") { sealed = true; done.resolve(); @@ -113,7 +128,7 @@ async function runTurn( await done.promise; clearTimeout(timeout); handlers.delete(conversationId); - return { state, deltas, sealed, error }; + return { state, telemetry, deltas, sealed, error }; } function toolChunksOf(state: TranscriptState) { @@ -178,6 +193,44 @@ async function main() { .join(""); record("turn 1 committed transcript has assistant text", committedText.length > 0); + // ─── Turn 1 telemetry: verify step metrics populated ─────────────────────── + const t1Turn = turnMetrics(t1.telemetry, textConv); + const t1StepCount = t1Turn?.steps.length ?? 0; + record("turn 1 telemetry accumulated steps", t1StepCount > 0, `${t1StepCount} step(s)`); + if (t1StepCount > 0) { + const s0 = stepMetrics(t1.telemetry, textConv, 0); + const hasTiming = s0?.genTotalMs !== undefined || s0?.ttftMs !== undefined; + if (hasTiming) { + record( + "turn 1 step 0 has timing metrics", + true, + `ttftMs=${s0?.ttftMs ?? "–"} decodeMs=${s0?.decodeMs ?? "–"} genTotalMs=${s0?.genTotalMs ?? "–"}`, + ); + } else { + note( + "turn 1 step 0 has no timing (backend may not have a clock) — telemetry path verified but no timing to assert", + ); + } + const hasTokens = s0?.usage?.outputTokens !== undefined; + if (hasTokens) { + record( + "turn 1 step 0 has token usage", + true, + `in=${s0?.usage?.inputTokens ?? "–"} out=${s0?.usage?.outputTokens ?? "–"}`, + ); + } else { + note( + "turn 1 step 0 has no usage (stepId may not have been on the usage event) — telemetry path verified", + ); + } + } + const t1Done = t1Turn?.wallMs; + if (t1Done !== undefined) { + record("turn 1 done event recorded wall-clock", true, `${t1Done}ms`); + } else { + note("turn 1 done.durationMs absent (backend clock unavailable)"); + } + // ─── Turn 2: tool-call batching ([email protected] stepId) ───────────────────────── console.log(`\n[live-probe] TURN 2 (tools): "${TOOL_PROMPT}"`); const toolConv = crypto.randomUUID(); @@ -185,6 +238,26 @@ async function main() { if (t2.error !== null) record("turn 2 had no chat.error", false, t2.error); record("turn 2 reached turn-sealed", t2.sealed); + // ─── Turn 2 telemetry: verify step + tool metrics ────────────────────────── + const t2Turn = turnMetrics(t2.telemetry, toolConv); + const t2StepCount = t2Turn?.steps.length ?? 0; + record("turn 2 telemetry accumulated steps", t2StepCount > 0, `${t2StepCount} step(s)`); + if (t2StepCount > 0) { + const s0 = stepMetrics(t2.telemetry, toolConv, 0); + if (s0?.toolDurationMs !== undefined && s0.toolDurationMs > 0) { + record("turn 2 step 0 has tool execution time", true, `toolDurationMs=${s0.toolDurationMs}`); + } else { + note("turn 2 step 0 has no toolDurationMs (tool-result.durationMs may be absent)"); + } + if (s0?.genTotalMs !== undefined) { + record("turn 2 step 0 has generation timing", true, `genTotalMs=${s0.genTotalMs}`); + } + } + const t2Done = t2Turn?.wallMs; + if (t2Done !== undefined) { + record("turn 2 done event recorded wall-clock", true, `${t2Done}ms`); + } + const liveTool = toolChunksOf(t2.state); const liveCalls = liveTool.filter((c) => c.chunk.type === "tool-call"); |