summaryrefslogtreecommitdiffhomepage
path: root/scripts
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-27 01:13:28 +0900
committerAdam Malczewski <[email protected]>2026-06-27 01:13:31 +0900
commit2fa03f8d7410c2b8d6be8e10ad088863e83d7177 (patch)
tree94e1923180ae38d571d34b578afecb0a18913c24 /scripts
parent80f99665034a0e510300793205c162fc7a46769f (diff)
parent08b12478636f4a5c86a1f3c40a843f2906b7c82f (diff)
downloaddispatch-web-2fa03f8d7410c2b8d6be8e10ad088863e83d7177.tar.gz
dispatch-web-2fa03f8d7410c2b8d6be8e10ad088863e83d7177.zip
Merge branch 'dev' into feature/heartbeat
# Conflicts: # src/app/App.svelte # src/app/store.svelte.ts # src/app/store.test.ts # src/features/workspaces/ui/WorkspaceCard.test.ts
Diffstat (limited to 'scripts')
-rw-r--r--scripts/live-probe-provider-retry.ts276
-rw-r--r--scripts/live-probe.ts600
-rw-r--r--scripts/probe-cache-warming.ts414
3 files changed, 645 insertions, 645 deletions
diff --git a/scripts/live-probe-provider-retry.ts b/scripts/live-probe-provider-retry.ts
index 22f1794..952163c 100644
--- a/scripts/live-probe-provider-retry.ts
+++ b/scripts/live-probe-provider-retry.ts
@@ -26,10 +26,10 @@ import type { SurfaceServerMessage } from "@dispatch/ui-contract";
import { createSurfaceSocket } from "../src/adapters/ws/index.ts";
import { parseServerMessage } from "../src/adapters/ws/logic.ts";
import {
- foldEvent,
- initialState,
- selectChunks,
- selectProviderRetry,
+ foldEvent,
+ initialState,
+ selectChunks,
+ selectProviderRetry,
} from "../src/core/chunks/index.ts";
const WS_URL = process.env.PROBE_WS ?? "ws://localhost:24205";
@@ -40,149 +40,149 @@ type ChatMsg = ChatDeltaMessage | ChatErrorMessage;
const checks: { name: string; ok: boolean; detail?: string }[] = [];
const record = (name: string, ok: boolean, detail?: string) => {
- checks.push({ name, ok, ...(detail !== undefined ? { detail } : {}) });
- console.log(` ${ok ? "✅" : "❌"} ${name}${detail ? ` — ${detail}` : ""}`);
+ checks.push({ name, ok, ...(detail !== undefined ? { detail } : {}) });
+ console.log(` ${ok ? "✅" : "❌"} ${name}${detail ? ` — ${detail}` : ""}`);
};
const fail = (msg: string): never => {
- console.error(`\n[probe] FATAL: ${msg}`);
- process.exit(1);
+ console.error(`\n[probe] FATAL: ${msg}`);
+ process.exit(1);
};
/** A chat.delta JSON frame carrying the given AgentEvent, exactly as the backend sends. */
function deltaFrame(event: ChatDeltaMessage["event"]): string {
- return JSON.stringify({ type: "chat.delta", event } satisfies ChatDeltaMessage);
+ return JSON.stringify({ type: "chat.delta", event } satisfies ChatDeltaMessage);
}
async function main() {
- console.log(`[probe] provider-retry seam · model=${MODEL} · WS=${WS_URL}\n`);
-
- // ─── 1. REGRESSION: a real text turn through the updated foldEvent ──────────
- // Routed by conversationId via a per-conv handler map (same pattern as live-probe.ts).
- const handlers = new Map<string, (msg: ChatMsg) => void>();
- const socket = createSurfaceSocket({
- url: WS_URL,
- onMessage: (_m: SurfaceServerMessage) => {},
- onChat: (msg: ChatMsg) => {
- const id = msg.type === "chat.error" ? msg.conversationId : msg.event.conversationId;
- const h = id !== undefined ? handlers.get(id) : undefined;
- h?.(msg);
- },
- });
- await new Promise((r) => setTimeout(r, 500));
-
- const conversationId = crypto.randomUUID();
- let state = initialState();
- let deltas = 0;
- let sealed = false;
- let error: string | null = null;
- const done = Promise.withResolvers<void>();
- handlers.set(conversationId, (msg) => {
- if (msg.type === "chat.error") {
- error = msg.message;
- done.resolve();
- return;
- }
- deltas++;
- state = foldEvent(state, msg.event);
- if (msg.event.type === "turn-sealed") {
- sealed = true;
- done.resolve();
- }
- });
-
- socket.send({ type: "chat.send", conversationId, message: PROMPT, model: MODEL });
- const timeout = setTimeout(() => done.resolve(), 90_000);
- await done.promise;
- clearTimeout(timeout);
- handlers.delete(conversationId);
-
- record(
- "regression: a real text turn sealed cleanly",
- sealed && error === null,
- `${deltas} deltas${error ? ` err=${error}` : ""}`,
- );
- record(
- "regression: providerRetry stayed NULL through a normal turn (no spurious banner)",
- selectProviderRetry(state) === null,
- );
-
- // ─── 2. PARSER + REDUCER SEAM: synthetic provider-retry through the REAL parser ─
- console.log("\n[probe] parser+reducer seam (synthetic provider-retry)");
- let s = initialState();
- s = foldEvent(s, { type: "turn-start", conversationId, turnId: "t1" });
-
- const retryJson = deltaFrame({
- type: "provider-retry",
- conversationId,
- turnId: "t1",
- attempt: 0,
- delayMs: 5000,
- message: 'HTTP 429: {"error":{"type":"overloaded_error","message":"overloaded"}}',
- code: "429",
- });
- const parsed1 = parseServerMessage(retryJson);
- record(
- "REAL parseServerMessage ACCEPTS a provider-retry chat.delta (not rejected as unknown)",
- parsed1 !== null && parsed1.type === "chat.delta" && parsed1.event.type === "provider-retry",
- parsed1 ? `event.type=${(parsed1 as { event: { type: string } }).event.type}` : "parsed=null",
- );
-
- if (parsed1 !== null && parsed1.type === "chat.delta") s = foldEvent(s, parsed1.event);
- const retry1 = selectProviderRetry(s);
- record(
- "foldEvent SETS providerRetry from the PARSED event",
- retry1 !== null && retry1.attempt === 0 && retry1.delayMs === 5000 && retry1.code === "429",
- retry1 ? `attempt=${retry1.attempt} delay=${retry1.delayMs}ms code=${retry1.code}` : "null",
- );
- record(
- "provider-retry adds NO chunk (never persisted — never pollutes the prompt)",
- selectChunks(s).length === 0,
- `${selectChunks(s).length} chunk(s)`,
- );
-
- const retry2Json = deltaFrame({
- type: "provider-retry",
- conversationId,
- turnId: "t1",
- attempt: 1,
- delayMs: 10000,
- message: "HTTP 429: still overloaded",
- code: "429",
- });
- const parsed2 = parseServerMessage(retry2Json);
- if (parsed2 !== null && parsed2.type === "chat.delta") s = foldEvent(s, parsed2.event);
- const retry2 = selectProviderRetry(s);
- record(
- "a 2nd provider-retry COALESCES (latest attempt + delay replaces previous)",
- retry2 !== null &&
- retry2.attempt === 1 &&
- retry2.delayMs === 10000 &&
- retry2.message === "HTTP 429: still overloaded",
- retry2 ? `attempt=${retry2.attempt} delay=${retry2.delayMs}ms` : "null",
- );
-
- const textJson = deltaFrame({
- type: "text-delta",
- conversationId,
- turnId: "t1",
- delta: "here is the reply",
- });
- const parsedText = parseServerMessage(textJson);
- if (parsedText !== null && parsedText.type === "chat.delta") s = foldEvent(s, parsedText.event);
- record(
- "a subsequent text-delta CLEARS the banner (retry succeeded → live reply)",
- selectProviderRetry(s) === null,
- );
- record(
- "…and the text-delta content DID land as a chunk (the reply streams normally after retries)",
- selectChunks(s).some((c) => c.chunk.type === "text"),
- );
-
- socket.close();
- const passed = checks.filter((c) => c.ok).length;
- const total = checks.length;
- console.log(`\n[probe] ${passed}/${total} checks passed`);
- process.exit(passed === total ? 0 : 1);
+ console.log(`[probe] provider-retry seam · model=${MODEL} · WS=${WS_URL}\n`);
+
+ // ─── 1. REGRESSION: a real text turn through the updated foldEvent ──────────
+ // Routed by conversationId via a per-conv handler map (same pattern as live-probe.ts).
+ const handlers = new Map<string, (msg: ChatMsg) => void>();
+ const socket = createSurfaceSocket({
+ url: WS_URL,
+ onMessage: (_m: SurfaceServerMessage) => {},
+ onChat: (msg: ChatMsg) => {
+ const id = msg.type === "chat.error" ? msg.conversationId : msg.event.conversationId;
+ const h = id !== undefined ? handlers.get(id) : undefined;
+ h?.(msg);
+ },
+ });
+ await new Promise((r) => setTimeout(r, 500));
+
+ const conversationId = crypto.randomUUID();
+ let state = initialState();
+ let deltas = 0;
+ let sealed = false;
+ let error: string | null = null;
+ const done = Promise.withResolvers<void>();
+ handlers.set(conversationId, (msg) => {
+ if (msg.type === "chat.error") {
+ error = msg.message;
+ done.resolve();
+ return;
+ }
+ deltas++;
+ state = foldEvent(state, msg.event);
+ if (msg.event.type === "turn-sealed") {
+ sealed = true;
+ done.resolve();
+ }
+ });
+
+ socket.send({ type: "chat.send", conversationId, message: PROMPT, model: MODEL });
+ const timeout = setTimeout(() => done.resolve(), 90_000);
+ await done.promise;
+ clearTimeout(timeout);
+ handlers.delete(conversationId);
+
+ record(
+ "regression: a real text turn sealed cleanly",
+ sealed && error === null,
+ `${deltas} deltas${error ? ` err=${error}` : ""}`,
+ );
+ record(
+ "regression: providerRetry stayed NULL through a normal turn (no spurious banner)",
+ selectProviderRetry(state) === null,
+ );
+
+ // ─── 2. PARSER + REDUCER SEAM: synthetic provider-retry through the REAL parser ─
+ console.log("\n[probe] parser+reducer seam (synthetic provider-retry)");
+ let s = initialState();
+ s = foldEvent(s, { type: "turn-start", conversationId, turnId: "t1" });
+
+ const retryJson = deltaFrame({
+ type: "provider-retry",
+ conversationId,
+ turnId: "t1",
+ attempt: 0,
+ delayMs: 5000,
+ message: 'HTTP 429: {"error":{"type":"overloaded_error","message":"overloaded"}}',
+ code: "429",
+ });
+ const parsed1 = parseServerMessage(retryJson);
+ record(
+ "REAL parseServerMessage ACCEPTS a provider-retry chat.delta (not rejected as unknown)",
+ parsed1 !== null && parsed1.type === "chat.delta" && parsed1.event.type === "provider-retry",
+ parsed1 ? `event.type=${(parsed1 as { event: { type: string } }).event.type}` : "parsed=null",
+ );
+
+ if (parsed1 !== null && parsed1.type === "chat.delta") s = foldEvent(s, parsed1.event);
+ const retry1 = selectProviderRetry(s);
+ record(
+ "foldEvent SETS providerRetry from the PARSED event",
+ retry1 !== null && retry1.attempt === 0 && retry1.delayMs === 5000 && retry1.code === "429",
+ retry1 ? `attempt=${retry1.attempt} delay=${retry1.delayMs}ms code=${retry1.code}` : "null",
+ );
+ record(
+ "provider-retry adds NO chunk (never persisted — never pollutes the prompt)",
+ selectChunks(s).length === 0,
+ `${selectChunks(s).length} chunk(s)`,
+ );
+
+ const retry2Json = deltaFrame({
+ type: "provider-retry",
+ conversationId,
+ turnId: "t1",
+ attempt: 1,
+ delayMs: 10000,
+ message: "HTTP 429: still overloaded",
+ code: "429",
+ });
+ const parsed2 = parseServerMessage(retry2Json);
+ if (parsed2 !== null && parsed2.type === "chat.delta") s = foldEvent(s, parsed2.event);
+ const retry2 = selectProviderRetry(s);
+ record(
+ "a 2nd provider-retry COALESCES (latest attempt + delay replaces previous)",
+ retry2 !== null &&
+ retry2.attempt === 1 &&
+ retry2.delayMs === 10000 &&
+ retry2.message === "HTTP 429: still overloaded",
+ retry2 ? `attempt=${retry2.attempt} delay=${retry2.delayMs}ms` : "null",
+ );
+
+ const textJson = deltaFrame({
+ type: "text-delta",
+ conversationId,
+ turnId: "t1",
+ delta: "here is the reply",
+ });
+ const parsedText = parseServerMessage(textJson);
+ if (parsedText !== null && parsedText.type === "chat.delta") s = foldEvent(s, parsedText.event);
+ record(
+ "a subsequent text-delta CLEARS the banner (retry succeeded → live reply)",
+ selectProviderRetry(s) === null,
+ );
+ record(
+ "…and the text-delta content DID land as a chunk (the reply streams normally after retries)",
+ selectChunks(s).some((c) => c.chunk.type === "text"),
+ );
+
+ socket.close();
+ const passed = checks.filter((c) => c.ok).length;
+ const total = checks.length;
+ console.log(`\n[probe] ${passed}/${total} checks passed`);
+ process.exit(passed === total ? 0 : 1);
}
main().catch((e) => fail(String(e)));
diff --git a/scripts/live-probe.ts b/scripts/live-probe.ts
index f44a136..6121eac 100644
--- a/scripts/live-probe.ts
+++ b/scripts/live-probe.ts
@@ -27,29 +27,29 @@
// browser has these natively; Bun does not). The product code is unchanged.
import "fake-indexeddb/auto";
import type {
- ChatDeltaMessage,
- ChatErrorMessage,
- ConversationHistoryResponse,
- ConversationMetricsResponse,
+ ChatDeltaMessage,
+ ChatErrorMessage,
+ ConversationHistoryResponse,
+ ConversationMetricsResponse,
} from "@dispatch/transport-contract";
import type { SurfaceServerMessage } from "@dispatch/ui-contract";
import { createIdbChunkStore } from "../src/adapters/idb/index.ts";
import { createSurfaceSocket } from "../src/adapters/ws/index.ts";
import {
- applyHistory,
- foldEvent,
- groupRenderedChunks,
- initialState,
- selectChunks,
- selectMessages,
- type TranscriptState,
+ applyHistory,
+ foldEvent,
+ groupRenderedChunks,
+ initialState,
+ selectChunks,
+ selectMessages,
+ type TranscriptState,
} from "../src/core/chunks/index.ts";
import {
- applyDurableMetrics,
- foldMetricsEvent,
- initialMetricsState,
- type MetricsState,
- selectOrderedTurnMetrics,
+ applyDurableMetrics,
+ foldMetricsEvent,
+ initialMetricsState,
+ type MetricsState,
+ selectOrderedTurnMetrics,
} from "../src/core/metrics/index.ts";
import { createConversationCache } from "../src/features/conversation-cache/index.ts";
@@ -58,51 +58,51 @@ const HTTP_BASE = process.env.PROBE_HTTP ?? "http://localhost:24203";
const MODEL = process.env.PROBE_MODEL ?? "opencode/deepseek-v4-flash";
const TEXT_PROMPT = process.env.PROBE_PROMPT ?? "Reply with exactly: hello from dispatch";
const TOOL_PROMPT =
- process.env.PROBE_TOOL_PROMPT ??
- "Make two tool calls AT THE SAME TIME in a single step (parallel tool calls). " +
- "For example, run two independent shell commands together: `echo alpha` and `echo beta`. " +
- "If you have no shell tool, invoke any two of your available read-only tools simultaneously.";
+ process.env.PROBE_TOOL_PROMPT ??
+ "Make two tool calls AT THE SAME TIME in a single step (parallel tool calls). " +
+ "For example, run two independent shell commands together: `echo alpha` and `echo beta`. " +
+ "If you have no shell tool, invoke any two of your available read-only tools simultaneously.";
const checks: { name: string; ok: boolean; detail?: string }[] = [];
const record = (name: string, ok: boolean, detail?: string) => {
- checks.push({ name, ok, ...(detail !== undefined ? { detail } : {}) });
- console.log(` ${ok ? "✅" : "❌"} ${name}${detail ? ` — ${detail}` : ""}`);
+ checks.push({ name, ok, ...(detail !== undefined ? { detail } : {}) });
+ console.log(` ${ok ? "✅" : "❌"} ${name}${detail ? ` — ${detail}` : ""}`);
};
const note = (msg: string) => console.log(` ℹ️ ${msg}`);
function fail(msg: string): never {
- console.error(`\n[live-probe] FATAL: ${msg}`);
- process.exit(1);
+ console.error(`\n[live-probe] FATAL: ${msg}`);
+ process.exit(1);
}
async function historySync(
- id: string,
- sinceSeq: number,
- window?: { limit?: number; beforeSeq?: number },
+ id: string,
+ sinceSeq: number,
+ window?: { limit?: number; beforeSeq?: number },
): Promise<ConversationHistoryResponse> {
- let url = `${HTTP_BASE}/conversations/${encodeURIComponent(id)}?sinceSeq=${sinceSeq}`;
- if (window?.limit !== undefined) url += `&limit=${window.limit}`;
- if (window?.beforeSeq !== undefined) url += `&beforeSeq=${window.beforeSeq}`;
- const res = await fetch(url, { headers: { Origin: "http://localhost:24204" } });
- if (!res.ok) fail(`history fetch ${res.status} for ${url}`);
- return (await res.json()) as ConversationHistoryResponse;
+ let url = `${HTTP_BASE}/conversations/${encodeURIComponent(id)}?sinceSeq=${sinceSeq}`;
+ if (window?.limit !== undefined) url += `&limit=${window.limit}`;
+ if (window?.beforeSeq !== undefined) url += `&beforeSeq=${window.beforeSeq}`;
+ const res = await fetch(url, { headers: { Origin: "http://localhost:24204" } });
+ if (!res.ok) fail(`history fetch ${res.status} for ${url}`);
+ return (await res.json()) as ConversationHistoryResponse;
}
/** Raw history GET that returns the status (for the CR-5 validation checks). */
async function historyStatus(id: string, query: string): Promise<number> {
- const url = `${HTTP_BASE}/conversations/${encodeURIComponent(id)}?${query}`;
- const res = await fetch(url, { headers: { Origin: "http://localhost:24204" } });
- await res.arrayBuffer(); // drain
- return res.status;
+ const url = `${HTTP_BASE}/conversations/${encodeURIComponent(id)}?${query}`;
+ const res = await fetch(url, { headers: { Origin: "http://localhost:24204" } });
+ await res.arrayBuffer(); // drain
+ return res.status;
}
/** Durable metrics fetch — returns the response, or the HTTP status when not OK
* (the endpoint is being implemented backend-side; the FE tolerates a 404). */
async function metricsSync(id: string): Promise<ConversationMetricsResponse | { status: number }> {
- const url = `${HTTP_BASE}/conversations/${encodeURIComponent(id)}/metrics`;
- const res = await fetch(url, { headers: { Origin: "http://localhost:24204" } });
- if (!res.ok) return { status: res.status };
- return (await res.json()) as ConversationMetricsResponse;
+ const url = `${HTTP_BASE}/conversations/${encodeURIComponent(id)}/metrics`;
+ const res = await fetch(url, { headers: { Origin: "http://localhost:24204" } });
+ if (!res.ok) return { status: res.status };
+ return (await res.json()) as ConversationMetricsResponse;
}
type ChatMsg = ChatDeltaMessage | ChatErrorMessage;
@@ -110,295 +110,295 @@ type Socket = ReturnType<typeof createSurfaceSocket>;
const handlers = new Map<string, (msg: ChatMsg) => void>();
function convOf(msg: ChatMsg): string | undefined {
- return msg.type === "chat.error" ? msg.conversationId : msg.event.conversationId;
+ return msg.type === "chat.error" ? msg.conversationId : msg.event.conversationId;
}
/** Drive one turn to turn-sealed (or error), folding events into a fresh state. */
async function runTurn(
- socket: Socket,
- conversationId: string,
- prompt: string,
+ socket: Socket,
+ conversationId: string,
+ prompt: string,
): Promise<{
- state: TranscriptState;
- metrics: MetricsState;
- deltas: number;
- sealed: boolean;
- error: string | null;
+ state: TranscriptState;
+ metrics: MetricsState;
+ deltas: number;
+ sealed: boolean;
+ error: string | null;
}> {
- let state = initialState();
- let metrics = initialMetricsState();
- let deltas = 0;
- let sealed = false;
- let error: string | null = null;
- const done = Promise.withResolvers<void>();
+ let state = initialState();
+ let metrics = initialMetricsState();
+ let deltas = 0;
+ let sealed = false;
+ let error: string | null = null;
+ const done = Promise.withResolvers<void>();
- handlers.set(conversationId, (msg) => {
- if (msg.type === "chat.error") {
- error = msg.message;
- done.resolve();
- return;
- }
- deltas++;
- state = foldEvent(state, msg.event);
- metrics = foldMetricsEvent(metrics, msg.event);
- if (msg.event.type === "turn-sealed") {
- sealed = true;
- done.resolve();
- }
- });
+ handlers.set(conversationId, (msg) => {
+ if (msg.type === "chat.error") {
+ error = msg.message;
+ done.resolve();
+ return;
+ }
+ deltas++;
+ state = foldEvent(state, msg.event);
+ metrics = foldMetricsEvent(metrics, msg.event);
+ if (msg.event.type === "turn-sealed") {
+ sealed = true;
+ done.resolve();
+ }
+ });
- socket.send({ type: "chat.send", conversationId, message: prompt, model: MODEL });
- const timeout = setTimeout(() => done.resolve(), 90_000);
- await done.promise;
- clearTimeout(timeout);
- handlers.delete(conversationId);
- return { state, metrics, deltas, sealed, error };
+ socket.send({ type: "chat.send", conversationId, message: prompt, model: MODEL });
+ const timeout = setTimeout(() => done.resolve(), 90_000);
+ await done.promise;
+ clearTimeout(timeout);
+ handlers.delete(conversationId);
+ return { state, metrics, deltas, sealed, error };
}
function toolChunksOf(state: TranscriptState) {
- return selectChunks(state).filter(
- (c) => c.chunk.type === "tool-call" || c.chunk.type === "tool-result",
- );
+ return selectChunks(state).filter(
+ (c) => c.chunk.type === "tool-call" || c.chunk.type === "tool-result",
+ );
}
async function main() {
- console.log(`[live-probe] model=${MODEL}`);
- console.log(`[live-probe] WS=${WS_URL} HTTP=${HTTP_BASE}\n`);
+ console.log(`[live-probe] model=${MODEL}`);
+ console.log(`[live-probe] WS=${WS_URL} HTTP=${HTTP_BASE}\n`);
- const cache = createConversationCache(createIdbChunkStore());
+ const cache = createConversationCache(createIdbChunkStore());
- let gotCatalog = false;
- const socket = createSurfaceSocket({
- url: WS_URL,
- onMessage: (m: SurfaceServerMessage) => {
- if (m.type === "catalog") {
- gotCatalog = true;
- console.log(` ↳ surface catalog: ${m.catalog.length} surface(s)`);
- }
- },
- onChat: (msg: ChatMsg) => {
- const id = convOf(msg);
- const h = id !== undefined ? handlers.get(id) : undefined;
- if (h) h(msg);
- },
- });
+ let gotCatalog = false;
+ const socket = createSurfaceSocket({
+ url: WS_URL,
+ onMessage: (m: SurfaceServerMessage) => {
+ if (m.type === "catalog") {
+ gotCatalog = true;
+ console.log(` ↳ surface catalog: ${m.catalog.length} surface(s)`);
+ }
+ },
+ onChat: (msg: ChatMsg) => {
+ const id = convOf(msg);
+ const h = id !== undefined ? handlers.get(id) : undefined;
+ if (h) h(msg);
+ },
+ });
- await new Promise((r) => setTimeout(r, 500));
- record("WS connected + surface catalog received", gotCatalog);
+ await new Promise((r) => setTimeout(r, 500));
+ record("WS connected + surface catalog received", gotCatalog);
- // ─── Turn 1: text streaming + cache + replay ────────────────────────────────
- console.log(`\n[live-probe] TURN 1 (text): "${TEXT_PROMPT}"`);
- const textConv = crypto.randomUUID();
- const t1 = await runTurn(socket, textConv, TEXT_PROMPT);
- if (t1.error !== null) record("turn 1 had no chat.error", false, t1.error);
- record("turn 1 received chat.delta events", t1.deltas > 0, `${t1.deltas} deltas`);
- record("turn 1 reached turn-sealed", t1.sealed);
+ // ─── Turn 1: text streaming + cache + replay ────────────────────────────────
+ console.log(`\n[live-probe] TURN 1 (text): "${TEXT_PROMPT}"`);
+ const textConv = crypto.randomUUID();
+ const t1 = await runTurn(socket, textConv, TEXT_PROMPT);
+ if (t1.error !== null) record("turn 1 had no chat.error", false, t1.error);
+ record("turn 1 received chat.delta events", t1.deltas > 0, `${t1.deltas} deltas`);
+ record("turn 1 reached turn-sealed", t1.sealed);
- let state = t1.state;
- const sinceSeq = await cache.sinceSeq(textConv);
- const hist = await historySync(textConv, sinceSeq);
- record(
- "turn 1 history endpoint returned chunks",
- hist.chunks.length > 0,
- `${hist.chunks.length} chunks, latestSeq=${hist.latestSeq}`,
- );
- const monotonic = hist.chunks.every((c, i) => i === 0 || c.seq > (hist.chunks[i - 1]?.seq ?? -1));
- record("turn 1 history chunks are seq-monotonic", monotonic);
- const merged = await cache.commit(textConv, hist.chunks);
- state = applyHistory(state, merged);
- record("turn 1 provisional superseded (sealedTurnId cleared)", state.sealedTurnId === null);
- const cached = await cache.load(textConv);
- record("turn 1 IndexedDB cache persisted the turn", cached.length === hist.chunks.length);
- const committedText = selectMessages(state)
- .filter((m) => m.role === "assistant")
- .flatMap((m) => m.chunks)
- .filter((c) => c.type === "text")
- .map((c) => (c as { text: string }).text)
- .join("");
- record("turn 1 committed transcript has assistant text", committedText.length > 0);
+ let state = t1.state;
+ const sinceSeq = await cache.sinceSeq(textConv);
+ const hist = await historySync(textConv, sinceSeq);
+ record(
+ "turn 1 history endpoint returned chunks",
+ hist.chunks.length > 0,
+ `${hist.chunks.length} chunks, latestSeq=${hist.latestSeq}`,
+ );
+ const monotonic = hist.chunks.every((c, i) => i === 0 || c.seq > (hist.chunks[i - 1]?.seq ?? -1));
+ record("turn 1 history chunks are seq-monotonic", monotonic);
+ const merged = await cache.commit(textConv, hist.chunks);
+ state = applyHistory(state, merged);
+ record("turn 1 provisional superseded (sealedTurnId cleared)", state.sealedTurnId === null);
+ const cached = await cache.load(textConv);
+ record("turn 1 IndexedDB cache persisted the turn", cached.length === hist.chunks.length);
+ const committedText = selectMessages(state)
+ .filter((m) => m.role === "assistant")
+ .flatMap((m) => m.chunks)
+ .filter((c) => c.type === "text")
+ .map((c) => (c as { text: string }).text)
+ .join("");
+ record("turn 1 committed transcript has assistant text", committedText.length > 0);
- // ─── CR-5: history windowing (?limit= / ?beforeSeq=, [email protected]) ───────
- const logLen = hist.chunks.length;
- record(
- "CR-5 seq origin: first chunk is seq 1 (1-based gap-free contract)",
- hist.chunks[0]?.seq === 1,
- `first seq=${hist.chunks[0]?.seq}`,
- );
- const win = await historySync(textConv, 0, { limit: 2 });
- record(
- "CR-5 ?limit=2 returns the NEWEST 2, ascending, latestSeq = window tail",
- win.chunks.length === Math.min(2, logLen) &&
- win.chunks[0]?.seq === Math.max(1, logLen - 1) &&
- win.chunks[win.chunks.length - 1]?.seq === logLen &&
- win.latestSeq === logLen,
- `seqs=[${win.chunks.map((c) => c.seq).join(",")}] latestSeq=${win.latestSeq}`,
- );
- const whole = await historySync(textConv, 0, { limit: 200 });
- record(
- "CR-5 ?limit= larger than the log returns everything (short-chat flow exact)",
- whole.chunks.length === logLen,
- `${whole.chunks.length}/${logLen} chunks`,
- );
- const oldestLoaded = win.chunks[0]?.seq ?? 0;
- if (oldestLoaded > 1) {
- const back = await historySync(textConv, 0, { beforeSeq: oldestLoaded, limit: 50 });
- record(
- "CR-5 ?beforeSeq= pages the older run (seq < bound, ascending from 1)",
- back.chunks.length === oldestLoaded - 1 &&
- back.chunks[0]?.seq === 1 &&
- back.chunks.every((c) => c.seq < oldestLoaded),
- `seqs=[${back.chunks.map((c) => c.seq).join(",")}]`,
- );
- }
- record("CR-5 limit=0 rejected with 400", (await historyStatus(textConv, "limit=0")) === 400);
- record(
- "CR-5 beforeSeq=-1 rejected with 400",
- (await historyStatus(textConv, "beforeSeq=-1")) === 400,
- );
+ // ─── CR-5: history windowing (?limit= / ?beforeSeq=, [email protected]) ───────
+ const logLen = hist.chunks.length;
+ record(
+ "CR-5 seq origin: first chunk is seq 1 (1-based gap-free contract)",
+ hist.chunks[0]?.seq === 1,
+ `first seq=${hist.chunks[0]?.seq}`,
+ );
+ const win = await historySync(textConv, 0, { limit: 2 });
+ record(
+ "CR-5 ?limit=2 returns the NEWEST 2, ascending, latestSeq = window tail",
+ win.chunks.length === Math.min(2, logLen) &&
+ win.chunks[0]?.seq === Math.max(1, logLen - 1) &&
+ win.chunks[win.chunks.length - 1]?.seq === logLen &&
+ win.latestSeq === logLen,
+ `seqs=[${win.chunks.map((c) => c.seq).join(",")}] latestSeq=${win.latestSeq}`,
+ );
+ const whole = await historySync(textConv, 0, { limit: 200 });
+ record(
+ "CR-5 ?limit= larger than the log returns everything (short-chat flow exact)",
+ whole.chunks.length === logLen,
+ `${whole.chunks.length}/${logLen} chunks`,
+ );
+ const oldestLoaded = win.chunks[0]?.seq ?? 0;
+ if (oldestLoaded > 1) {
+ const back = await historySync(textConv, 0, { beforeSeq: oldestLoaded, limit: 50 });
+ record(
+ "CR-5 ?beforeSeq= pages the older run (seq < bound, ascending from 1)",
+ back.chunks.length === oldestLoaded - 1 &&
+ back.chunks[0]?.seq === 1 &&
+ back.chunks.every((c) => c.seq < oldestLoaded),
+ `seqs=[${back.chunks.map((c) => c.seq).join(",")}]`,
+ );
+ }
+ record("CR-5 limit=0 rejected with 400", (await historyStatus(textConv, "limit=0")) === 400);
+ record(
+ "CR-5 beforeSeq=-1 rejected with 400",
+ (await historyStatus(textConv, "beforeSeq=-1")) === 400,
+ );
- // ─── Metrics: LIVE token + timing ([email protected] usage/step-complete/done) ──────
- // (TurnMetricsEntry is `{ turnId, steps, total }` — the turn aggregate lives on
- // `total`, present once the live `done` folded.)
- const liveTurns = selectOrderedTurnMetrics(t1.metrics);
- const m1 = liveTurns[0];
- const m1Total = m1?.total ?? null;
- record(
- "turn 1 LIVE metrics: a turn with output tokens",
- m1Total !== null && m1Total.usage.outputTokens > 0,
- m1Total
- ? `in=${m1Total.usage.inputTokens} out=${m1Total.usage.outputTokens} steps=${m1?.steps.length}`
- : "no finalized turn total",
- );
- if (m1 !== undefined) {
- const anyGen = m1.steps.some((s) => s.genTotalMs !== undefined);
- const anyTtft = m1.steps.some((s) => s.ttftMs !== undefined);
- note(
- `live timing: durationMs=${m1Total?.durationMs ?? "—"}, ` +
- `genTotalMs present=${anyGen}, ttftMs present=${anyTtft}`,
- );
- record(
- "turn 1 LIVE metrics carries timing (durationMs or step genTotalMs)",
- m1Total?.durationMs !== undefined || anyGen,
- "requires the backend runtime to have a clock",
- );
- }
+ // ─── Metrics: LIVE token + timing ([email protected] usage/step-complete/done) ──────
+ // (TurnMetricsEntry is `{ turnId, steps, total }` — the turn aggregate lives on
+ // `total`, present once the live `done` folded.)
+ const liveTurns = selectOrderedTurnMetrics(t1.metrics);
+ const m1 = liveTurns[0];
+ const m1Total = m1?.total ?? null;
+ record(
+ "turn 1 LIVE metrics: a turn with output tokens",
+ m1Total !== null && m1Total.usage.outputTokens > 0,
+ m1Total
+ ? `in=${m1Total.usage.inputTokens} out=${m1Total.usage.outputTokens} steps=${m1?.steps.length}`
+ : "no finalized turn total",
+ );
+ if (m1 !== undefined) {
+ const anyGen = m1.steps.some((s) => s.genTotalMs !== undefined);
+ const anyTtft = m1.steps.some((s) => s.ttftMs !== undefined);
+ note(
+ `live timing: durationMs=${m1Total?.durationMs ?? "—"}, ` +
+ `genTotalMs present=${anyGen}, ttftMs present=${anyTtft}`,
+ );
+ record(
+ "turn 1 LIVE metrics carries timing (durationMs or step genTotalMs)",
+ m1Total?.durationMs !== undefined || anyGen,
+ "requires the backend runtime to have a clock",
+ );
+ }
- // ─── Metrics: DURABLE endpoint (GET /conversations/:id/metrics) ──────────────
- const dm = await metricsSync(textConv);
- if ("status" in dm) {
- note(
- `durable /metrics not available yet (HTTP ${dm.status}) — FE degrades to live-only, as designed`,
- );
- record(
- "durable /metrics is implemented OR gracefully absent (404)",
- dm.status === 404 || dm.status === 405,
- `HTTP ${dm.status}`,
- );
- } else {
- record(
- "durable /metrics returned TurnMetrics[]",
- Array.isArray(dm.turns),
- `${dm.turns.length} turn(s)`,
- );
- const durableMerged = selectOrderedTurnMetrics(
- applyDurableMetrics(initialMetricsState(), dm.turns),
- );
- const d1 = durableMerged[0];
- const d1Total = d1?.total ?? null;
- record(
- "durable /metrics turn has token usage",
- d1Total !== null && d1Total.usage.outputTokens > 0,
- d1Total ? `out=${d1Total.usage.outputTokens} steps=${d1?.steps.length}` : "no turn total",
- );
- }
+ // ─── Metrics: DURABLE endpoint (GET /conversations/:id/metrics) ──────────────
+ const dm = await metricsSync(textConv);
+ if ("status" in dm) {
+ note(
+ `durable /metrics not available yet (HTTP ${dm.status}) — FE degrades to live-only, as designed`,
+ );
+ record(
+ "durable /metrics is implemented OR gracefully absent (404)",
+ dm.status === 404 || dm.status === 405,
+ `HTTP ${dm.status}`,
+ );
+ } else {
+ record(
+ "durable /metrics returned TurnMetrics[]",
+ Array.isArray(dm.turns),
+ `${dm.turns.length} turn(s)`,
+ );
+ const durableMerged = selectOrderedTurnMetrics(
+ applyDurableMetrics(initialMetricsState(), dm.turns),
+ );
+ const d1 = durableMerged[0];
+ const d1Total = d1?.total ?? null;
+ record(
+ "durable /metrics turn has token usage",
+ d1Total !== null && d1Total.usage.outputTokens > 0,
+ d1Total ? `out=${d1Total.usage.outputTokens} steps=${d1?.steps.length}` : "no turn total",
+ );
+ }
- // ─── Turn 2: tool-call batching ([email protected] stepId) ─────────────────────────
- console.log(`\n[live-probe] TURN 2 (tools): "${TOOL_PROMPT}"`);
- const toolConv = crypto.randomUUID();
- const t2 = await runTurn(socket, toolConv, TOOL_PROMPT);
- if (t2.error !== null) record("turn 2 had no chat.error", false, t2.error);
- record("turn 2 reached turn-sealed", t2.sealed);
+ // ─── Turn 2: tool-call batching ([email protected] stepId) ─────────────────────────
+ console.log(`\n[live-probe] TURN 2 (tools): "${TOOL_PROMPT}"`);
+ const toolConv = crypto.randomUUID();
+ const t2 = await runTurn(socket, toolConv, TOOL_PROMPT);
+ if (t2.error !== null) record("turn 2 had no chat.error", false, t2.error);
+ record("turn 2 reached turn-sealed", t2.sealed);
- const liveTool = toolChunksOf(t2.state);
- const liveCalls = liveTool.filter((c) => c.chunk.type === "tool-call");
+ const liveTool = toolChunksOf(t2.state);
+ const liveCalls = liveTool.filter((c) => c.chunk.type === "tool-call");
- if (liveCalls.length === 0) {
- note(
- "INCONCLUSIVE: the model issued no tool calls this run — cannot verify stepId grouping live. " +
- "Re-run with a stronger PROBE_TOOL_PROMPT or one tailored to the backend's tool set.",
- );
- record("turn 2 tool-call batching (live)", true, "skipped — no tool calls issued");
- } else {
- // Every live tool chunk must carry stepId (foldEvent copies it from the event).
- const allLiveHaveStep = liveTool.every(
- (c) =>
- (c.chunk.type === "tool-call" || c.chunk.type === "tool-result") &&
- typeof c.chunk.stepId === "string" &&
- c.chunk.stepId.length > 0,
- );
- record(
- "turn 2 every LIVE tool event carries stepId",
- allLiveHaveStep,
- `${liveCalls.length} call(s), ${liveTool.length - liveCalls.length} result(s)`,
- );
+ if (liveCalls.length === 0) {
+ note(
+ "INCONCLUSIVE: the model issued no tool calls this run — cannot verify stepId grouping live. " +
+ "Re-run with a stronger PROBE_TOOL_PROMPT or one tailored to the backend's tool set.",
+ );
+ record("turn 2 tool-call batching (live)", true, "skipped — no tool calls issued");
+ } else {
+ // Every live tool chunk must carry stepId (foldEvent copies it from the event).
+ const allLiveHaveStep = liveTool.every(
+ (c) =>
+ (c.chunk.type === "tool-call" || c.chunk.type === "tool-result") &&
+ typeof c.chunk.stepId === "string" &&
+ c.chunk.stepId.length > 0,
+ );
+ record(
+ "turn 2 every LIVE tool event carries stepId",
+ allLiveHaveStep,
+ `${liveCalls.length} call(s), ${liveTool.length - liveCalls.length} result(s)`,
+ );
- const liveGroups = groupRenderedChunks(selectChunks(t2.state));
- const liveBatches = liveGroups.filter((g) => g.kind === "tool-batch");
- const distinctSteps = new Set(
- liveCalls.map((c) => (c.chunk.type === "tool-call" ? c.chunk.stepId : undefined)),
- );
- note(
- `live grouping: ${liveCalls.length} call(s) across ${distinctSteps.size} step(s) → ` +
- `${liveBatches.length} batch group(s)`,
- );
- if (liveBatches.length > 0) {
- record(
- "turn 2 grouping produced a parallel batch (2+ calls in one step)",
- true,
- `${liveBatches.length} batch(es)`,
- );
- } else {
- note(
- "the model used tools but did NOT parallelize (each call its own step) — stepId is verified, " +
- "but no multi-call batch occurred to render as a list this run.",
- );
- }
+ const liveGroups = groupRenderedChunks(selectChunks(t2.state));
+ const liveBatches = liveGroups.filter((g) => g.kind === "tool-batch");
+ const distinctSteps = new Set(
+ liveCalls.map((c) => (c.chunk.type === "tool-call" ? c.chunk.stepId : undefined)),
+ );
+ note(
+ `live grouping: ${liveCalls.length} call(s) across ${distinctSteps.size} step(s) → ` +
+ `${liveBatches.length} batch group(s)`,
+ );
+ if (liveBatches.length > 0) {
+ record(
+ "turn 2 grouping produced a parallel batch (2+ calls in one step)",
+ true,
+ `${liveBatches.length} batch(es)`,
+ );
+ } else {
+ note(
+ "the model used tools but did NOT parallelize (each call its own step) — stepId is verified, " +
+ "but no multi-call batch occurred to render as a list this run.",
+ );
+ }
- // Replay path: persisted tool chunks must also carry chunk.stepId.
- const histTool = await historySync(toolConv, 0);
- const replayTool = histTool.chunks.filter(
- (c) => c.chunk.type === "tool-call" || c.chunk.type === "tool-result",
- );
- const allReplayHaveStep = replayTool.every(
- (c) =>
- (c.chunk.type === "tool-call" || c.chunk.type === "tool-result") &&
- typeof c.chunk.stepId === "string" &&
- c.chunk.stepId.length > 0,
- );
- record(
- "turn 2 every REPLAYED tool chunk carries chunk.stepId",
- replayTool.length > 0 && allReplayHaveStep,
- `${replayTool.length} tool chunk(s) in history`,
- );
+ // Replay path: persisted tool chunks must also carry chunk.stepId.
+ const histTool = await historySync(toolConv, 0);
+ const replayTool = histTool.chunks.filter(
+ (c) => c.chunk.type === "tool-call" || c.chunk.type === "tool-result",
+ );
+ const allReplayHaveStep = replayTool.every(
+ (c) =>
+ (c.chunk.type === "tool-call" || c.chunk.type === "tool-result") &&
+ typeof c.chunk.stepId === "string" &&
+ c.chunk.stepId.length > 0,
+ );
+ record(
+ "turn 2 every REPLAYED tool chunk carries chunk.stepId",
+ replayTool.length > 0 && allReplayHaveStep,
+ `${replayTool.length} tool chunk(s) in history`,
+ );
- // Grouping on the authoritative replayed history matches the live shape.
- const replayState = applyHistory(initialState(), await cache.commit(toolConv, histTool.chunks));
- const replayBatches = groupRenderedChunks(selectChunks(replayState)).filter(
- (g) => g.kind === "tool-batch",
- );
- record(
- "turn 2 replay grouping matches live (batch count)",
- replayBatches.length === liveBatches.length,
- `live=${liveBatches.length} replay=${replayBatches.length}`,
- );
- }
+ // Grouping on the authoritative replayed history matches the live shape.
+ const replayState = applyHistory(initialState(), await cache.commit(toolConv, histTool.chunks));
+ const replayBatches = groupRenderedChunks(selectChunks(replayState)).filter(
+ (g) => g.kind === "tool-batch",
+ );
+ record(
+ "turn 2 replay grouping matches live (batch count)",
+ replayBatches.length === liveBatches.length,
+ `live=${liveBatches.length} replay=${replayBatches.length}`,
+ );
+ }
- socket.close();
+ socket.close();
- const passed = checks.filter((c) => c.ok).length;
- const total = checks.length;
- console.log(`\n[live-probe] ${passed}/${total} checks passed`);
- process.exit(passed === total ? 0 : 1);
+ const passed = checks.filter((c) => c.ok).length;
+ const total = checks.length;
+ console.log(`\n[live-probe] ${passed}/${total} checks passed`);
+ process.exit(passed === total ? 0 : 1);
}
main().catch((e) => fail(String(e)));
diff --git a/scripts/probe-cache-warming.ts b/scripts/probe-cache-warming.ts
index 470e43b..1bf1f9b 100644
--- a/scripts/probe-cache-warming.ts
+++ b/scripts/probe-cache-warming.ts
@@ -14,9 +14,9 @@
* bun scripts/probe-cache-warming.ts
*/
import type {
- ChatDeltaMessage,
- ChatErrorMessage,
- CloseConversationResponse,
+ ChatDeltaMessage,
+ ChatErrorMessage,
+ CloseConversationResponse,
} from "@dispatch/transport-contract";
import type { SurfaceServerMessage, SurfaceSpec } from "@dispatch/ui-contract";
import { createSurfaceSocket } from "../src/adapters/ws/index.ts";
@@ -28,17 +28,17 @@ const SURFACE_ID = "cache-warming";
const checks: { name: string; ok: boolean }[] = [];
const record = (name: string, ok: boolean, detail?: string) => {
- checks.push({ name, ok });
- console.log(` ${ok ? "✅" : "❌"} ${name}${detail ? ` — ${detail}` : ""}`);
+ checks.push({ name, ok });
+ console.log(` ${ok ? "✅" : "❌"} ${name}${detail ? ` — ${detail}` : ""}`);
};
const log = (msg: string) => console.log(`[${new Date().toISOString().slice(11, 19)}] ${msg}`);
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
function summarize(spec: SurfaceSpec | null): string {
- const c = parseControls(spec);
- const next =
- c.nextWarmAt === null ? "null" : `${Math.round((c.nextWarmAt - Date.now()) / 1000)}s`;
- return `enabled=${c.enabled} interval=${c.intervalSeconds}s lastPct=${c.lastPct} next=${next} lastWarmAt=${c.lastWarmAt}`;
+ const c = parseControls(spec);
+ const next =
+ c.nextWarmAt === null ? "null" : `${Math.round((c.nextWarmAt - Date.now()) / 1000)}s`;
+ return `enabled=${c.enabled} interval=${c.intervalSeconds}s lastPct=${c.lastPct} next=${next} lastWarmAt=${c.lastWarmAt}`;
}
let catalog: { id: string; scope?: string }[] = [];
@@ -49,229 +49,229 @@ let specWaiter: (() => void) | null = null;
const chatHandlers = new Map<string, (msg: ChatDeltaMessage | ChatErrorMessage) => void>();
const socket = createSurfaceSocket({
- url: WS_URL,
- onMessage: (m: SurfaceServerMessage) => {
- if (m.type === "catalog") {
- catalog = [...m.catalog];
- log(`catalog: ${m.catalog.map((e) => `${e.id}(scope=${e.scope ?? "—"})`).join(", ")}`);
- } else if (m.type === "surface") {
- latestSpec = m.spec;
- latestSpecConv = m.conversationId;
- log(`surface(initial) conv=${m.conversationId ?? "—"}: ${summarize(m.spec)}`);
- specWaiter?.();
- } else if (m.type === "update") {
- if (m.update.surfaceId !== SURFACE_ID) return;
- latestSpec = m.update.spec;
- latestSpecConv = m.update.conversationId;
- log(`update conv=${m.update.conversationId ?? "—"}: ${summarize(m.update.spec)}`);
- specWaiter?.();
- } else if (m.type === "error") {
- log(`surface ERROR: ${m.surfaceId ?? "—"}: ${m.message}`);
- }
- },
- onChat: (msg) => {
- const id = msg.type === "chat.error" ? msg.conversationId : msg.event.conversationId;
- if (id !== undefined) chatHandlers.get(id)?.(msg);
- },
+ url: WS_URL,
+ onMessage: (m: SurfaceServerMessage) => {
+ if (m.type === "catalog") {
+ catalog = [...m.catalog];
+ log(`catalog: ${m.catalog.map((e) => `${e.id}(scope=${e.scope ?? "—"})`).join(", ")}`);
+ } else if (m.type === "surface") {
+ latestSpec = m.spec;
+ latestSpecConv = m.conversationId;
+ log(`surface(initial) conv=${m.conversationId ?? "—"}: ${summarize(m.spec)}`);
+ specWaiter?.();
+ } else if (m.type === "update") {
+ if (m.update.surfaceId !== SURFACE_ID) return;
+ latestSpec = m.update.spec;
+ latestSpecConv = m.update.conversationId;
+ log(`update conv=${m.update.conversationId ?? "—"}: ${summarize(m.update.spec)}`);
+ specWaiter?.();
+ } else if (m.type === "error") {
+ log(`surface ERROR: ${m.surfaceId ?? "—"}: ${m.message}`);
+ }
+ },
+ onChat: (msg) => {
+ const id = msg.type === "chat.error" ? msg.conversationId : msg.event.conversationId;
+ if (id !== undefined) chatHandlers.get(id)?.(msg);
+ },
});
/** Wait for the next surface/update message (or time out). */
function nextSpec(timeoutMs: number): Promise<boolean> {
- return new Promise((resolve) => {
- const t = setTimeout(() => {
- specWaiter = null;
- resolve(false);
- }, timeoutMs);
- specWaiter = () => {
- clearTimeout(t);
- specWaiter = null;
- resolve(true);
- };
- });
+ return new Promise((resolve) => {
+ const t = setTimeout(() => {
+ specWaiter = null;
+ resolve(false);
+ }, timeoutMs);
+ specWaiter = () => {
+ clearTimeout(t);
+ specWaiter = null;
+ resolve(true);
+ };
+ });
}
async function runTinyTurn(conversationId: string, prompt: string): Promise<boolean> {
- const done = Promise.withResolvers<boolean>();
- chatHandlers.set(conversationId, (msg) => {
- if (msg.type === "chat.error") {
- log(`chat.error: ${msg.message}`);
- done.resolve(false);
- } else if (msg.event.type === "turn-sealed") {
- done.resolve(true);
- }
- });
- socket.send({ type: "chat.send", conversationId, message: prompt });
- const t = setTimeout(() => done.resolve(false), 90_000);
- const ok = await done.promise;
- clearTimeout(t);
- chatHandlers.delete(conversationId);
- return ok;
+ const done = Promise.withResolvers<boolean>();
+ chatHandlers.set(conversationId, (msg) => {
+ if (msg.type === "chat.error") {
+ log(`chat.error: ${msg.message}`);
+ done.resolve(false);
+ } else if (msg.event.type === "turn-sealed") {
+ done.resolve(true);
+ }
+ });
+ socket.send({ type: "chat.send", conversationId, message: prompt });
+ const t = setTimeout(() => done.resolve(false), 90_000);
+ const ok = await done.promise;
+ clearTimeout(t);
+ chatHandlers.delete(conversationId);
+ return ok;
}
function invoke(actionId: string, conversationId: string, payload?: unknown): void {
- socket.send(
- payload === undefined
- ? { type: "invoke", surfaceId: SURFACE_ID, actionId, conversationId }
- : { type: "invoke", surfaceId: SURFACE_ID, actionId, payload, conversationId },
- );
+ socket.send(
+ payload === undefined
+ ? { type: "invoke", surfaceId: SURFACE_ID, actionId, conversationId }
+ : { type: "invoke", surfaceId: SURFACE_ID, actionId, payload, conversationId },
+ );
}
async function main() {
- await sleep(600);
- record(
- "catalog includes cache-warming with scope=conversation",
- catalog.some((e) => e.id === SURFACE_ID && e.scope === "conversation"),
- );
+ await sleep(600);
+ record(
+ "catalog includes cache-warming with scope=conversation",
+ catalog.some((e) => e.id === SURFACE_ID && e.scope === "conversation"),
+ );
- // ── A: the DRAFT/new-tab path — subscribe with NO conversationId ───────────
- log("PHASE A: subscribe with NO conversationId (draft / new tab)");
- socket.send({ type: "subscribe", surfaceId: SURFACE_ID });
- await nextSpec(3000);
- record(
- "draft subscribe → degenerate spec (no toggle parsed)",
- !parseControls(latestSpec).enabled,
- );
- socket.send({ type: "unsubscribe", surfaceId: SURFACE_ID });
- await sleep(300);
+ // ── A: the DRAFT/new-tab path — subscribe with NO conversationId ───────────
+ log("PHASE A: subscribe with NO conversationId (draft / new tab)");
+ socket.send({ type: "subscribe", surfaceId: SURFACE_ID });
+ await nextSpec(3000);
+ record(
+ "draft subscribe → degenerate spec (no toggle parsed)",
+ !parseControls(latestSpec).enabled,
+ );
+ socket.send({ type: "unsubscribe", surfaceId: SURFACE_ID });
+ await sleep(300);
- // ── B: a FRESH conversation defaults OFF (CR-4a) + echo (CR-4d) ────────────
- const conv = crypto.randomUUID();
- log(`PHASE B: creating conversation ${conv}`);
- if (!(await runTinyTurn(conv, "Reply with exactly: ok"))) {
- log("FATAL: could not create a conversation");
- process.exit(1);
- }
- socket.send({ type: "subscribe", surfaceId: SURFACE_ID, conversationId: conv });
- await nextSpec(3000);
- const fresh = parseControls(latestSpec);
- record("CR-4d: initial surface message echoes conversationId", latestSpecConv === conv);
- record("CR-4a: fresh conversation defaults to warming OFF", fresh.enabled === false);
- record("CR-4a: nothing scheduled while off (nextWarmAt null)", fresh.nextWarmAt === null);
+ // ── B: a FRESH conversation defaults OFF (CR-4a) + echo (CR-4d) ────────────
+ const conv = crypto.randomUUID();
+ log(`PHASE B: creating conversation ${conv}`);
+ if (!(await runTinyTurn(conv, "Reply with exactly: ok"))) {
+ log("FATAL: could not create a conversation");
+ process.exit(1);
+ }
+ socket.send({ type: "subscribe", surfaceId: SURFACE_ID, conversationId: conv });
+ await nextSpec(3000);
+ const fresh = parseControls(latestSpec);
+ record("CR-4d: initial surface message echoes conversationId", latestSpecConv === conv);
+ record("CR-4a: fresh conversation defaults to warming OFF", fresh.enabled === false);
+ record("CR-4a: nothing scheduled while off (nextWarmAt null)", fresh.nextWarmAt === null);
- // ── C: opt in + 10s interval → repeated warms, FUTURE nextWarmAt (CR-4b) ───
- log("PHASE C: toggling warming ON");
- const toggleId = fresh.toggleActionId;
- if (toggleId === null) {
- record("toggle action present", false);
- process.exit(1);
- }
- invoke(toggleId, conv);
- await nextSpec(3000);
- let c = parseControls(latestSpec);
- record("toggle-on update arrived (enabled)", c.enabled === true);
- record(
- "CR-4b: enable schedules a FUTURE nextWarmAt",
- c.nextWarmAt !== null && c.nextWarmAt > Date.now(),
- );
+ // ── C: opt in + 10s interval → repeated warms, FUTURE nextWarmAt (CR-4b) ───
+ log("PHASE C: toggling warming ON");
+ const toggleId = fresh.toggleActionId;
+ if (toggleId === null) {
+ record("toggle action present", false);
+ process.exit(1);
+ }
+ invoke(toggleId, conv);
+ await nextSpec(3000);
+ let c = parseControls(latestSpec);
+ record("toggle-on update arrived (enabled)", c.enabled === true);
+ record(
+ "CR-4b: enable schedules a FUTURE nextWarmAt",
+ c.nextWarmAt !== null && c.nextWarmAt > Date.now(),
+ );
- const setIntervalId = c.setIntervalActionId;
- if (setIntervalId !== null) {
- log("PHASE C: set-interval = 10s");
- invoke(setIntervalId, conv, 10);
- await nextSpec(3000);
- c = parseControls(latestSpec);
- record(
- "set-interval update: interval=10 + FUTURE nextWarmAt",
- c.intervalSeconds === 10 && c.nextWarmAt !== null && c.nextWarmAt > Date.now(),
- );
- }
+ const setIntervalId = c.setIntervalActionId;
+ if (setIntervalId !== null) {
+ log("PHASE C: set-interval = 10s");
+ invoke(setIntervalId, conv, 10);
+ await nextSpec(3000);
+ c = parseControls(latestSpec);
+ record(
+ "set-interval update: interval=10 + FUTURE nextWarmAt",
+ c.intervalSeconds === 10 && c.nextWarmAt !== null && c.nextWarmAt > Date.now(),
+ );
+ }
- log("PHASE C: waiting up to 45s for 2 automatic warms…");
- const deadline = Date.now() + 45_000;
- let lastSeen = c.lastWarmAt;
- let warms = 0;
- let allFuture = true;
- while (Date.now() < deadline && warms < 2) {
- await nextSpec(Math.max(1, deadline - Date.now()));
- const now = parseControls(latestSpec);
- if (now.lastWarmAt !== null && now.lastWarmAt !== lastSeen) {
- lastSeen = now.lastWarmAt;
- warms++;
- const future = now.nextWarmAt !== null && now.nextWarmAt > Date.now() - 1000;
- if (!future) allFuture = false;
- log(
- ` automatic warm #${warms}: pct=${now.lastPct} retention=${now.retentionPct} ` +
- `nextWarmAt ${future ? "FUTURE" : "STALE/PAST"}`,
- );
- }
- }
- record("automatic warms repeat (2 observed @10s)", warms >= 2, `${warms} warm(s)`);
- record("CR-4b: every post-warm update carries a FUTURE nextWarmAt", warms >= 2 && allFuture);
+ log("PHASE C: waiting up to 45s for 2 automatic warms…");
+ const deadline = Date.now() + 45_000;
+ let lastSeen = c.lastWarmAt;
+ let warms = 0;
+ let allFuture = true;
+ while (Date.now() < deadline && warms < 2) {
+ await nextSpec(Math.max(1, deadline - Date.now()));
+ const now = parseControls(latestSpec);
+ if (now.lastWarmAt !== null && now.lastWarmAt !== lastSeen) {
+ lastSeen = now.lastWarmAt;
+ warms++;
+ const future = now.nextWarmAt !== null && now.nextWarmAt > Date.now() - 1000;
+ if (!future) allFuture = false;
+ log(
+ ` automatic warm #${warms}: pct=${now.lastPct} retention=${now.retentionPct} ` +
+ `nextWarmAt ${future ? "FUTURE" : "STALE/PAST"}`,
+ );
+ }
+ }
+ record("automatic warms repeat (2 observed @10s)", warms >= 2, `${warms} warm(s)`);
+ record("CR-4b: every post-warm update carries a FUTURE nextWarmAt", warms >= 2 && allFuture);
- // ── D: close mid-turn → abort + warming disabled (CR-4c) ───────────────────
- log("PHASE D: starting a long turn, then closing the conversation mid-turn…");
- const seenDone = Promise.withResolvers<string>(); // resolves with done.reason
- const seenSealed = Promise.withResolvers<void>();
- let turnStarted = false;
- const started = Promise.withResolvers<void>();
- chatHandlers.set(conv, (msg) => {
- if (msg.type === "chat.error") {
- log(`chat.error: ${msg.message}`);
- return;
- }
- const ev = msg.event;
- if (ev.type === "turn-start") {
- turnStarted = true;
- started.resolve();
- } else if (ev.type === "done") {
- seenDone.resolve(ev.reason);
- } else if (ev.type === "turn-sealed") {
- seenSealed.resolve();
- }
- });
- socket.send({
- type: "chat.send",
- conversationId: conv,
- message:
- "Write a detailed 1000-word essay about the history of computing. Take your time and be thorough.",
- });
- const startTimeout = setTimeout(() => started.resolve(), 15_000);
- await started.promise;
- clearTimeout(startTimeout);
- record("turn started (watcher saw turn-start)", turnStarted);
- await sleep(1000); // let it generate a moment
+ // ── D: close mid-turn → abort + warming disabled (CR-4c) ───────────────────
+ log("PHASE D: starting a long turn, then closing the conversation mid-turn…");
+ const seenDone = Promise.withResolvers<string>(); // resolves with done.reason
+ const seenSealed = Promise.withResolvers<void>();
+ let turnStarted = false;
+ const started = Promise.withResolvers<void>();
+ chatHandlers.set(conv, (msg) => {
+ if (msg.type === "chat.error") {
+ log(`chat.error: ${msg.message}`);
+ return;
+ }
+ const ev = msg.event;
+ if (ev.type === "turn-start") {
+ turnStarted = true;
+ started.resolve();
+ } else if (ev.type === "done") {
+ seenDone.resolve(ev.reason);
+ } else if (ev.type === "turn-sealed") {
+ seenSealed.resolve();
+ }
+ });
+ socket.send({
+ type: "chat.send",
+ conversationId: conv,
+ message:
+ "Write a detailed 1000-word essay about the history of computing. Take your time and be thorough.",
+ });
+ const startTimeout = setTimeout(() => started.resolve(), 15_000);
+ await started.promise;
+ clearTimeout(startTimeout);
+ record("turn started (watcher saw turn-start)", turnStarted);
+ await sleep(1000); // let it generate a moment
- const res = await fetch(`${HTTP_BASE}/conversations/${encodeURIComponent(conv)}/close`, {
- method: "POST",
- headers: { Origin: "http://localhost:24204" },
- });
- record("POST /conversations/:id/close → 200", res.ok, `HTTP ${res.status}`);
- const body = (await res.json()) as CloseConversationResponse;
- record("close aborted the in-flight turn (abortedTurn)", body.abortedTurn === true);
+ const res = await fetch(`${HTTP_BASE}/conversations/${encodeURIComponent(conv)}/close`, {
+ method: "POST",
+ headers: { Origin: "http://localhost:24204" },
+ });
+ record("POST /conversations/:id/close → 200", res.ok, `HTTP ${res.status}`);
+ const body = (await res.json()) as CloseConversationResponse;
+ record("close aborted the in-flight turn (abortedTurn)", body.abortedTurn === true);
- const doneReason = await Promise.race([seenDone.promise, sleep(15_000).then(() => "(timeout)")]);
- record('watcher received done with reason "aborted"', doneReason === "aborted", doneReason);
- const sealed = await Promise.race([
- seenSealed.promise.then(() => true),
- sleep(15_000).then(() => false),
- ]);
- record("turn sealed normally after abort", sealed);
- chatHandlers.delete(conv);
+ const doneReason = await Promise.race([seenDone.promise, sleep(15_000).then(() => "(timeout)")]);
+ record('watcher received done with reason "aborted"', doneReason === "aborted", doneReason);
+ const sealed = await Promise.race([
+ seenSealed.promise.then(() => true),
+ sleep(15_000).then(() => false),
+ ]);
+ record("turn sealed normally after abort", sealed);
+ chatHandlers.delete(conv);
- // The close also pushed a surface update: warming disabled + unscheduled.
- await sleep(1500);
- const closed = parseControls(latestSpec);
- record(
- "CR-4c: close disabled warming + cleared the schedule",
- closed.enabled === false && closed.nextWarmAt === null,
- summarize(latestSpec),
- );
+ // The close also pushed a surface update: warming disabled + unscheduled.
+ await sleep(1500);
+ const closed = parseControls(latestSpec);
+ record(
+ "CR-4c: close disabled warming + cleared the schedule",
+ closed.enabled === false && closed.nextWarmAt === null,
+ summarize(latestSpec),
+ );
- // Idempotency: closing again (now idle) succeeds with abortedTurn false.
- const res2 = await fetch(`${HTTP_BASE}/conversations/${encodeURIComponent(conv)}/close`, {
- method: "POST",
- headers: { Origin: "http://localhost:24204" },
- });
- const body2 = (await res2.json()) as CloseConversationResponse;
- record("close is idempotent (200 + abortedTurn:false)", res2.ok && body2.abortedTurn === false);
+ // Idempotency: closing again (now idle) succeeds with abortedTurn false.
+ const res2 = await fetch(`${HTTP_BASE}/conversations/${encodeURIComponent(conv)}/close`, {
+ method: "POST",
+ headers: { Origin: "http://localhost:24204" },
+ });
+ const body2 = (await res2.json()) as CloseConversationResponse;
+ record("close is idempotent (200 + abortedTurn:false)", res2.ok && body2.abortedTurn === false);
- socket.close();
- const passed = checks.filter((x) => x.ok).length;
- console.log(`\n[probe-cache-warming] ${passed}/${checks.length} checks passed`);
- process.exit(passed === checks.length ? 0 : 1);
+ socket.close();
+ const passed = checks.filter((x) => x.ok).length;
+ console.log(`\n[probe-cache-warming] ${passed}/${checks.length} checks passed`);
+ process.exit(passed === checks.length ? 0 : 1);
}
main().catch((e) => {
- console.error(`[probe] FATAL: ${e}`);
- process.exit(1);
+ console.error(`[probe] FATAL: ${e}`);
+ process.exit(1);
});