summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorAdam Malczewski <[email protected]>2026-06-05 15:16:14 +0900
committerAdam Malczewski <[email protected]>2026-06-05 15:16:14 +0900
commit20c6c675a11b887c603be5ff08165cb182c7db65 (patch)
treee57ef71daa6cb7c5c73a4bd9af5be6cafe9d2158
parent4d94c530406567791dbe4ab06c838a83c2e26023 (diff)
downloaddispatch-20c6c675a11b887c603be5ff08165cb182c7db65.tar.gz
dispatch-20c6c675a11b887c603be5ff08165cb182c7db65.zip
feat(observability): Phase B — trace-store (SQLite) + out-of-process collector + trace CLI (345 tests)
trace-store (bun:sqlite): records+bodies schema (thin/fat split), idempotent insertRecords (FNV-1a id + INSERT OR IGNORE), getTurn/getBody, pure renderEasyView (D8 timeline skeleton), trace CLI. Its own DB, isolated from storage-sqlite. observability-collector: out-of-process bin — tail journal -> splitLines/drainOnce -> trace-store.insertRecords; offset sidecar; at-least-once + idempotent; fail-safe; clean SIGINT/SIGTERM drain. Build-config (orchestrator): root tsconfig refs; both excluded from vitest + added to test:bun (bun:sqlite); bun install. Verified: tsc -b clean, 345 tests (273 vitest + 72 bun), biome 0 warnings/0 infos. Pipeline proven end-to-end: app -> journal -> collector -> SQLite -> 'trace <turnId>' easy-view. Known follow-up (next commit): kernel spans are flat (parent=ROOT) — run-turn nesting fix.
-rw-r--r--bun.lock19
-rw-r--r--package.json2
-rw-r--r--packages/observability-collector/package.json12
-rw-r--r--packages/observability-collector/src/collector.test.ts282
-rw-r--r--packages/observability-collector/src/collector.ts162
-rw-r--r--packages/observability-collector/src/index.ts9
-rw-r--r--packages/observability-collector/src/main.ts87
-rw-r--r--packages/observability-collector/tsconfig.json6
-rw-r--r--packages/trace-store/package.json11
-rw-r--r--packages/trace-store/src/cli.ts16
-rw-r--r--packages/trace-store/src/easy-view.test.ts360
-rw-r--r--packages/trace-store/src/easy-view.ts205
-rw-r--r--packages/trace-store/src/index.ts3
-rw-r--r--packages/trace-store/src/store.test.ts224
-rw-r--r--packages/trace-store/src/store.ts254
-rw-r--r--packages/trace-store/tsconfig.json6
-rw-r--r--tasks.md24
-rw-r--r--tsconfig.json2
-rw-r--r--vitest.config.ts7
19 files changed, 1685 insertions, 6 deletions
diff --git a/bun.lock b/bun.lock
index d5446fe..03f0e4d 100644
--- a/bun.lock
+++ b/bun.lock
@@ -51,6 +51,14 @@
"name": "@dispatch/kernel",
"version": "0.0.0",
},
+ "packages/observability-collector": {
+ "name": "@dispatch/observability-collector",
+ "version": "0.0.0",
+ "dependencies": {
+ "@dispatch/kernel": "workspace:*",
+ "@dispatch/trace-store": "workspace:*",
+ },
+ },
"packages/provider-openai-compat": {
"name": "@dispatch/provider-openai-compat",
"version": "0.0.0",
@@ -80,6 +88,13 @@
"@dispatch/kernel": "workspace:*",
},
},
+ "packages/trace-store": {
+ "name": "@dispatch/trace-store",
+ "version": "0.0.0",
+ "dependencies": {
+ "@dispatch/kernel": "workspace:*",
+ },
+ },
"packages/transport-http": {
"name": "@dispatch/transport-http",
"version": "0.0.0",
@@ -119,6 +134,8 @@
"@dispatch/kernel": ["@dispatch/kernel@workspace:packages/kernel"],
+ "@dispatch/observability-collector": ["@dispatch/observability-collector@workspace:packages/observability-collector"],
+
"@dispatch/provider-openai-compat": ["@dispatch/provider-openai-compat@workspace:packages/provider-openai-compat"],
"@dispatch/session-orchestrator": ["@dispatch/session-orchestrator@workspace:packages/session-orchestrator"],
@@ -127,6 +144,8 @@
"@dispatch/tool-read-file": ["@dispatch/tool-read-file@workspace:packages/tool-read-file"],
+ "@dispatch/trace-store": ["@dispatch/trace-store@workspace:packages/trace-store"],
+
"@dispatch/transport-http": ["@dispatch/transport-http@workspace:packages/transport-http"],
"@esbuild/aix-ppc64": ["@esbuild/[email protected]", "", { "os": "aix", "cpu": "ppc64" }, "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg=="],
diff --git a/package.json b/package.json
index 8d19268..d46ea7f 100644
--- a/package.json
+++ b/package.json
@@ -11,7 +11,7 @@
"test": "vitest run",
"test:watch": "vitest",
"typecheck": "tsc -b --pretty",
- "test:bun": "bun test packages/storage-sqlite/src",
+ "test:bun": "bun test packages/storage-sqlite/src packages/trace-store/src packages/observability-collector/src",
"test:all": "bun run test && bun run test:bun",
"dev": "bun packages/host-bin/src/main.ts"
},
diff --git a/packages/observability-collector/package.json b/packages/observability-collector/package.json
new file mode 100644
index 0000000..b744dc7
--- /dev/null
+++ b/packages/observability-collector/package.json
@@ -0,0 +1,12 @@
+{
+ "name": "@dispatch/observability-collector",
+ "version": "0.0.0",
+ "type": "module",
+ "private": true,
+ "main": "dist/index.js",
+ "types": "dist/index.d.ts",
+ "dependencies": {
+ "@dispatch/kernel": "workspace:*",
+ "@dispatch/trace-store": "workspace:*"
+ }
+}
diff --git a/packages/observability-collector/src/collector.test.ts b/packages/observability-collector/src/collector.test.ts
new file mode 100644
index 0000000..51b3e42
--- /dev/null
+++ b/packages/observability-collector/src/collector.test.ts
@@ -0,0 +1,282 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { LogRecord } from "@dispatch/kernel";
+import { createTraceStore } from "@dispatch/trace-store";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { drainOnce, readOffset, splitLines, writeOffset } from "./collector.js";
+
+// --- Fixtures ---
+
+const log1: LogRecord = {
+ kind: "log",
+ level: "info",
+ msg: "first",
+ timestamp: 1700000000000,
+ extensionId: "ext-1",
+ turnId: "turn-1",
+};
+
+const log2: LogRecord = {
+ kind: "log",
+ level: "warn",
+ msg: "second",
+ timestamp: 1700000000100,
+ extensionId: "ext-1",
+ turnId: "turn-1",
+};
+
+const spanOpen: LogRecord = {
+ kind: "span-open",
+ spanId: "span-1",
+ name: "step",
+ timestamp: 1700000000200,
+ extensionId: "ext-1",
+ turnId: "turn-1",
+};
+
+const spanClose: LogRecord = {
+ kind: "span-close",
+ spanId: "span-1",
+ name: "step",
+ timestamp: 1700000000500,
+ durationMs: 300,
+ status: "ok",
+ extensionId: "ext-1",
+ turnId: "turn-1",
+};
+
+function toNdjson(records: LogRecord[]): string {
+ return `${records.map((r) => JSON.stringify(r)).join("\n")}\n`;
+}
+
+// --- splitLines (pure) ---
+
+describe("splitLines", () => {
+ it("splits multiple lines", () => {
+ const { lines, remainder } = splitLines("a\nb\nc\n");
+ expect(lines).toEqual(["a", "b", "c"]);
+ expect(remainder).toBe("");
+ });
+
+ it("holds a torn last line as remainder (no trailing newline)", () => {
+ const { lines, remainder } = splitLines("a\nb\nc");
+ expect(lines).toEqual(["a", "b"]);
+ expect(remainder).toBe("c");
+ });
+
+ it("returns empty lines and empty remainder for empty buffer", () => {
+ const { lines, remainder } = splitLines("");
+ expect(lines).toEqual([]);
+ expect(remainder).toBe("");
+ });
+
+ it("handles a single complete line", () => {
+ const { lines, remainder } = splitLines("hello\n");
+ expect(lines).toEqual(["hello"]);
+ expect(remainder).toBe("");
+ });
+
+ it("handles a single incomplete line", () => {
+ const { lines, remainder } = splitLines("hello");
+ expect(lines).toEqual([]);
+ expect(remainder).toBe("hello");
+ });
+
+ it("handles consecutive newlines (empty lines)", () => {
+ const { lines, remainder } = splitLines("a\n\nb\n");
+ expect(lines).toEqual(["a", "", "b"]);
+ expect(remainder).toBe("");
+ });
+
+ it("handles only newlines", () => {
+ const { lines, remainder } = splitLines("\n\n\n");
+ expect(lines).toEqual(["", "", ""]);
+ expect(remainder).toBe("");
+ });
+});
+
+// --- drainOnce (integration with real temp files + in-memory store) ---
+
+let tmpDir: string;
+
+beforeEach(() => {
+ tmpDir = mkdtempSync(join(tmpdir(), "collector-test-"));
+});
+
+afterEach(() => {
+ rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("drainOnce", () => {
+ it("reads N NDJSON records from journal and inserts into store", () => {
+ const journalPath = join(tmpDir, "journal.log");
+ writeFileSync(journalPath, toNdjson([log1, log2, spanOpen]));
+
+ const store = createTraceStore({ path: ":memory:" });
+ const result = drainOnce({ journalPath, offset: 0, store });
+
+ expect(result.newOffset).toBeGreaterThan(0);
+ const turn = store.getTurn("turn-1");
+ expect(turn).toHaveLength(3);
+ store.close();
+ });
+
+ it("returns offset at EOF after draining all records", () => {
+ const journalPath = join(tmpDir, "journal.log");
+ const content = toNdjson([log1, log2]);
+ writeFileSync(journalPath, content);
+
+ const store = createTraceStore({ path: ":memory:" });
+ const result = drainOnce({ journalPath, offset: 0, store });
+
+ expect(result.newOffset).toBe(Buffer.byteLength(content, "utf8"));
+ store.close();
+ });
+
+ it("appends more lines and drains only new records from newOffset", () => {
+ const journalPath = join(tmpDir, "journal.log");
+ const initial = toNdjson([log1]);
+ writeFileSync(journalPath, initial);
+
+ const store = createTraceStore({ path: ":memory:" });
+ const r1 = drainOnce({ journalPath, offset: 0, store });
+ expect(store.getTurn("turn-1")).toHaveLength(1);
+
+ // Append more
+ const additional = toNdjson([log2, spanOpen]);
+ writeFileSync(journalPath, initial + additional, { flag: "a" });
+
+ const r2 = drainOnce({ journalPath, offset: r1.newOffset, store });
+ expect(r2.newOffset).toBeGreaterThan(r1.newOffset);
+ expect(store.getTurn("turn-1")).toHaveLength(3);
+ store.close();
+ });
+
+ it("holds a torn last line (no trailing newline) until newline arrives", () => {
+ const journalPath = join(tmpDir, "journal.log");
+ const full = toNdjson([log1]);
+ // Write log1 + partial log2 (no trailing newline)
+ const partial = JSON.stringify(log2);
+ writeFileSync(journalPath, full + partial);
+
+ const store = createTraceStore({ path: ":memory:" });
+ const r1 = drainOnce({ journalPath, offset: 0, store });
+
+ // Only log1 should be inserted; partial log2 is held
+ expect(store.getTurn("turn-1")).toHaveLength(1);
+
+ // Now append newline to complete log2
+ writeFileSync(journalPath, `${full + partial}\n`, { flag: "a" });
+ drainOnce({ journalPath, offset: r1.newOffset, store });
+
+ expect(store.getTurn("turn-1")).toHaveLength(2);
+ store.close();
+ });
+
+ it("skips malformed lines (warn, no throw)", () => {
+ const journalPath = join(tmpDir, "journal.log");
+ const good = JSON.stringify(log1);
+ const bad = "this is not valid json{{{";
+ const good2 = JSON.stringify(log2);
+ writeFileSync(journalPath, `${good}\n${bad}\n${good2}\n`);
+
+ const store = createTraceStore({ path: ":memory:" });
+ const warnCalls: unknown[][] = [];
+ const origWarn = console.warn;
+ console.warn = (...args: unknown[]) => warnCalls.push(args);
+
+ try {
+ drainOnce({ journalPath, offset: 0, store });
+ } finally {
+ console.warn = origWarn;
+ }
+
+ const turn = store.getTurn("turn-1");
+ expect(turn).toHaveLength(2);
+ expect(warnCalls.length).toBeGreaterThan(0);
+ expect(String(warnCalls[0]?.[0])).toContain("malformed line");
+ store.close();
+ });
+
+ it("re-draining from offset 0 inserts no duplicates (idempotent)", () => {
+ const journalPath = join(tmpDir, "journal.log");
+ writeFileSync(journalPath, toNdjson([log1, log2, spanOpen, spanClose]));
+
+ const store = createTraceStore({ path: ":memory:" });
+
+ // Drain twice from offset 0
+ drainOnce({ journalPath, offset: 0, store });
+ drainOnce({ journalPath, offset: 0, store });
+
+ const turn = store.getTurn("turn-1");
+ // trace-store uses INSERT OR IGNORE, so no duplicates
+ expect(turn).toHaveLength(4);
+ store.close();
+ });
+
+ it("returns same offset when journal is empty", () => {
+ const journalPath = join(tmpDir, "journal.log");
+ writeFileSync(journalPath, "");
+
+ const store = createTraceStore({ path: ":memory:" });
+ const result = drainOnce({ journalPath, offset: 0, store });
+
+ expect(result.newOffset).toBe(0);
+ store.close();
+ });
+
+ it("returns same offset when no new content past offset", () => {
+ const journalPath = join(tmpDir, "journal.log");
+ writeFileSync(journalPath, toNdjson([log1]));
+
+ const store = createTraceStore({ path: ":memory:" });
+ const r1 = drainOnce({ journalPath, offset: 0, store });
+ const r2 = drainOnce({ journalPath, offset: r1.newOffset, store });
+
+ expect(r2.newOffset).toBe(r1.newOffset);
+ store.close();
+ });
+
+ it("returns same offset when journal file does not exist", () => {
+ const journalPath = join(tmpDir, "nonexistent.log");
+ const store = createTraceStore({ path: ":memory:" });
+ const result = drainOnce({ journalPath, offset: 0, store });
+
+ expect(result.newOffset).toBe(0);
+ store.close();
+ });
+});
+
+// --- Offset persistence ---
+
+describe("readOffset / writeOffset", () => {
+ it("returns 0 when sidecar file does not exist", () => {
+ expect(readOffset(join(tmpDir, "nope.offset"))).toBe(0);
+ });
+
+ it("reads back a persisted offset", () => {
+ const path = join(tmpDir, "test.offset");
+ writeOffset(path, 42);
+ expect(readOffset(path)).toBe(42);
+ });
+
+ it("overwrites previous offset", () => {
+ const path = join(tmpDir, "test.offset");
+ writeOffset(path, 100);
+ writeOffset(path, 200);
+ expect(readOffset(path)).toBe(200);
+ });
+
+ it("returns 0 for non-numeric content", () => {
+ const path = join(tmpDir, "bad.offset");
+ writeFileSync(path, "not-a-number");
+ expect(readOffset(path)).toBe(0);
+ });
+
+ it("returns 0 for negative content", () => {
+ const path = join(tmpDir, "neg.offset");
+ writeFileSync(path, "-5");
+ expect(readOffset(path)).toBe(0);
+ });
+});
diff --git a/packages/observability-collector/src/collector.ts b/packages/observability-collector/src/collector.ts
new file mode 100644
index 0000000..549090b
--- /dev/null
+++ b/packages/observability-collector/src/collector.ts
@@ -0,0 +1,162 @@
+import type { LogRecord } from "@dispatch/kernel";
+import type { TraceStore } from "@dispatch/trace-store";
+
+// --- Pure core (no I/O) ---
+
+/**
+ * Split a buffer on newline boundaries. Returns complete lines and the
+ * trailing partial (no newline yet) as remainder. A torn last line is
+ * NOT parsed until its newline arrives.
+ */
+export function splitLines(buffer: string): { lines: string[]; remainder: string } {
+ const lines: string[] = [];
+ let start = 0;
+
+ for (let i = 0; i < buffer.length; i++) {
+ if (buffer[i] === "\n") {
+ lines.push(buffer.slice(start, i));
+ start = i + 1;
+ }
+ }
+
+ const remainder = buffer.slice(start);
+ return { lines, remainder };
+}
+
+// --- Drain step (the unit of work) ---
+
+export interface DrainOpts {
+ readonly journalPath: string;
+ readonly offset: number;
+ readonly store: TraceStore;
+}
+
+export interface DrainResult {
+ readonly newOffset: number;
+}
+
+/**
+ * Read bytes from offset to EOF, split into NDJSON lines, parse each
+ * complete line into a LogRecord, skip + warn on malformed lines,
+ * insert the batch into the store, return the new offset past the
+ * consumed complete lines (excluding any held remainder).
+ */
+export function drainOnce(opts: DrainOpts): DrainResult {
+ const { journalPath, offset, store } = opts;
+
+ let content: string;
+ try {
+ content = readFileFromOffset(journalPath, offset);
+ } catch {
+ return { newOffset: offset };
+ }
+
+ if (content.length === 0) {
+ return { newOffset: offset };
+ }
+
+ const { lines } = splitLines(content);
+
+ if (lines.length === 0) {
+ return { newOffset: offset };
+ }
+
+ const records: LogRecord[] = [];
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (trimmed.length === 0) continue;
+ try {
+ const parsed: LogRecord = JSON.parse(trimmed);
+ records.push(parsed);
+ } catch (err) {
+ console.warn("[observability-collector] skipping malformed line:", err);
+ }
+ }
+
+ if (records.length > 0) {
+ store.insertRecords(records);
+ }
+
+ const consumedBytes = Buffer.byteLength(
+ lines.join("\n") + (lines.length > 0 ? "\n" : ""),
+ "utf8",
+ );
+ return { newOffset: offset + consumedBytes };
+}
+
+// --- Offset persistence ---
+
+/**
+ * Read the resume offset from a sidecar file. Returns 0 if missing.
+ */
+export function readOffset(sidecarPath: string): number {
+ try {
+ const content = readFileUtf8(sidecarPath).trim();
+ const parsed = Number(content);
+ return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0;
+ } catch {
+ return 0;
+ }
+}
+
+/**
+ * Persist the resume offset to a sidecar file.
+ */
+export function writeOffset(sidecarPath: string, offset: number): void {
+ writeFileUtf8(sidecarPath, String(offset));
+}
+
+// --- I/O abstraction (injected at edges, default = real fs) ---
+
+export interface FsOps {
+ readonly readFileFromOffset: (path: string, offset: number) => string;
+ readonly readFileUtf8: (path: string) => string;
+ readonly writeFileUtf8: (path: string, data: string) => void;
+}
+
+let fsOps: FsOps = createDefaultFsOps();
+
+export function setFsOps(ops: FsOps): void {
+ fsOps = ops;
+}
+
+export function resetFsOps(): void {
+ fsOps = createDefaultFsOps();
+}
+
+function readFileFromOffset(path: string, offset: number): string {
+ return fsOps.readFileFromOffset(path, offset);
+}
+
+function readFileUtf8(path: string): string {
+ return fsOps.readFileUtf8(path);
+}
+
+function writeFileUtf8(path: string, data: string): void {
+ fsOps.writeFileUtf8(path, data);
+}
+
+function createDefaultFsOps(): FsOps {
+ const fs = require("node:fs") as typeof import("node:fs");
+ return {
+ readFileFromOffset(path: string, offset: number): string {
+ const fd = fs.openSync(path, "r");
+ try {
+ const stat = fs.fstatSync(fd);
+ const bytesToRead = stat.size - offset;
+ if (bytesToRead <= 0) return "";
+ const buf = Buffer.alloc(bytesToRead);
+ fs.readSync(fd, buf, 0, bytesToRead, offset);
+ return buf.toString("utf8");
+ } finally {
+ fs.closeSync(fd);
+ }
+ },
+ readFileUtf8(path: string): string {
+ return fs.readFileSync(path, "utf8");
+ },
+ writeFileUtf8(path: string, data: string): void {
+ fs.writeFileSync(path, data, "utf8");
+ },
+ };
+}
diff --git a/packages/observability-collector/src/index.ts b/packages/observability-collector/src/index.ts
new file mode 100644
index 0000000..4a38600
--- /dev/null
+++ b/packages/observability-collector/src/index.ts
@@ -0,0 +1,9 @@
+export type { DrainOpts, DrainResult, FsOps } from "./collector.js";
+export {
+ drainOnce,
+ readOffset,
+ resetFsOps,
+ setFsOps,
+ splitLines,
+ writeOffset,
+} from "./collector.js";
diff --git a/packages/observability-collector/src/main.ts b/packages/observability-collector/src/main.ts
new file mode 100644
index 0000000..3418e09
--- /dev/null
+++ b/packages/observability-collector/src/main.ts
@@ -0,0 +1,87 @@
+import { createTraceStore } from "@dispatch/trace-store";
+import { drainOnce, readOffset, writeOffset } from "./collector.js";
+
+// --- Argv parsing ---
+
+interface CliArgs {
+ readonly journal: string;
+ readonly db: string;
+ readonly interval: number;
+}
+
+function parseArgs(argv: string[]): CliArgs {
+ let journal = "";
+ let db = "./.dispatch-data/traces.db";
+ let interval = 250;
+
+ for (let i = 0; i < argv.length; i++) {
+ const arg = argv[i];
+ if (arg === "--journal" && i + 1 < argv.length) {
+ journal = argv[i + 1] ?? "";
+ i++;
+ } else if (arg === "--db" && i + 1 < argv.length) {
+ db = argv[i + 1] ?? db;
+ i++;
+ } else if (arg === "--interval" && i + 1 < argv.length) {
+ const val = Number(argv[i + 1]);
+ if (Number.isFinite(val) && val > 0) interval = val;
+ i++;
+ }
+ }
+
+ if (!journal) {
+ console.error(
+ "Usage: observability-collector --journal <path> [--db <path>] [--interval <ms>]",
+ );
+ process.exit(1);
+ }
+
+ return { journal, db, interval };
+}
+
+// --- Main loop ---
+
+async function main(): Promise<void> {
+ const args = parseArgs(process.argv.slice(2));
+ const sidecarPath = `${args.journal}.collector-offset`;
+ const store = createTraceStore({ path: args.db });
+
+ let offset = readOffset(sidecarPath);
+
+ let shuttingDown = false;
+
+ function onSignal(): void {
+ if (shuttingDown) return;
+ shuttingDown = true;
+ }
+
+ process.on("SIGINT", onSignal);
+ process.on("SIGTERM", onSignal);
+
+ while (!shuttingDown) {
+ const result = drainOnce({ journalPath: args.journal, offset, store });
+ if (result.newOffset !== offset) {
+ offset = result.newOffset;
+ writeOffset(sidecarPath, offset);
+ }
+ await sleep(args.interval);
+ }
+
+ // Final drain on shutdown
+ const finalResult = drainOnce({ journalPath: args.journal, offset, store });
+ if (finalResult.newOffset !== offset) {
+ writeOffset(sidecarPath, finalResult.newOffset);
+ }
+
+ store.close();
+ process.exit(0);
+}
+
+function sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+main().catch((err) => {
+ console.error("[observability-collector] fatal:", err);
+ process.exit(1);
+});
diff --git a/packages/observability-collector/tsconfig.json b/packages/observability-collector/tsconfig.json
new file mode 100644
index 0000000..7bfb36e
--- /dev/null
+++ b/packages/observability-collector/tsconfig.json
@@ -0,0 +1,6 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": { "rootDir": "src", "outDir": "dist", "composite": true },
+ "include": ["src/**/*.ts"],
+ "references": [{ "path": "../kernel" }, { "path": "../trace-store" }]
+}
diff --git a/packages/trace-store/package.json b/packages/trace-store/package.json
new file mode 100644
index 0000000..9809efd
--- /dev/null
+++ b/packages/trace-store/package.json
@@ -0,0 +1,11 @@
+{
+ "name": "@dispatch/trace-store",
+ "version": "0.0.0",
+ "type": "module",
+ "private": true,
+ "main": "dist/index.js",
+ "types": "dist/index.d.ts",
+ "dependencies": {
+ "@dispatch/kernel": "workspace:*"
+ }
+}
diff --git a/packages/trace-store/src/cli.ts b/packages/trace-store/src/cli.ts
new file mode 100644
index 0000000..9092c91
--- /dev/null
+++ b/packages/trace-store/src/cli.ts
@@ -0,0 +1,16 @@
+import { createTraceStore } from "./store.js";
+
+const turnId = process.argv[2];
+if (turnId === undefined) {
+ console.error("Usage: bun packages/trace-store/src/cli.ts <turnId> [dbPath]");
+ process.exit(1);
+}
+
+const dbPath = process.argv[3] ?? process.env.TRACE_DB_PATH ?? "./.dispatch-data/traces.db";
+const store = createTraceStore({ path: dbPath });
+try {
+ const output = store.easyView(turnId);
+ console.log(output);
+} finally {
+ store.close();
+}
diff --git a/packages/trace-store/src/easy-view.test.ts b/packages/trace-store/src/easy-view.test.ts
new file mode 100644
index 0000000..3ebc814
--- /dev/null
+++ b/packages/trace-store/src/easy-view.test.ts
@@ -0,0 +1,360 @@
+import type { LogRecord } from "@dispatch/kernel";
+import { describe, expect, it } from "vitest";
+import { formatDuration, renderEasyView } from "./easy-view.js";
+
+describe("renderEasyView", () => {
+ it("returns empty string for empty records", () => {
+ expect(renderEasyView([])).toBe("");
+ });
+
+ it("renders a single span with no children", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "span-open",
+ spanId: "s1",
+ name: "step",
+ timestamp: 1000,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ {
+ kind: "span-close",
+ spanId: "s1",
+ name: "step",
+ timestamp: 2800,
+ durationMs: 1800,
+ status: "ok",
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ expect(renderEasyView(records)).toBe("- step 1.8s");
+ });
+
+ it("renders a span with nested log records", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "span-open",
+ spanId: "s1",
+ name: "step",
+ timestamp: 1000,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ {
+ kind: "log",
+ level: "info",
+ msg: "thinking...",
+ timestamp: 1200,
+ extensionId: "ext",
+ turnId: "t1",
+ parentSpanId: "s1",
+ },
+ {
+ kind: "span-close",
+ spanId: "s1",
+ name: "step",
+ timestamp: 2800,
+ durationMs: 1800,
+ status: "ok",
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ const expected = ["- step 1.8s", " - thinking..."].join("\n");
+ expect(renderEasyView(records)).toBe(expected);
+ });
+
+ it("renders nested spans with indentation", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "span-open",
+ spanId: "parent",
+ name: "step",
+ timestamp: 1000,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ {
+ kind: "span-open",
+ spanId: "child",
+ name: "tool.call",
+ timestamp: 1100,
+ extensionId: "ext",
+ turnId: "t1",
+ parentSpanId: "parent",
+ },
+ {
+ kind: "log",
+ level: "info",
+ msg: "executing",
+ timestamp: 1200,
+ extensionId: "ext",
+ turnId: "t1",
+ parentSpanId: "child",
+ },
+ {
+ kind: "span-close",
+ spanId: "child",
+ name: "tool.call",
+ timestamp: 1500,
+ durationMs: 400,
+ status: "ok",
+ extensionId: "ext",
+ turnId: "t1",
+ parentSpanId: "parent",
+ },
+ {
+ kind: "span-close",
+ spanId: "parent",
+ name: "step",
+ timestamp: 2800,
+ durationMs: 1800,
+ status: "ok",
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ const expected = ["- step 1.8s", " - tool.call 400ms", " - executing"].join("\n");
+ expect(renderEasyView(records)).toBe(expected);
+ });
+
+ it("renders a span-open without matching span-close as (open)", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "span-open",
+ spanId: "s-open",
+ name: "step",
+ timestamp: 1000,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ expect(renderEasyView(records)).toBe("- step (open)");
+ });
+
+ it("renders root-level log records without a parent span", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "log",
+ level: "info",
+ msg: "top-level",
+ timestamp: 500,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ {
+ kind: "log",
+ level: "warn",
+ msg: "a warning",
+ timestamp: 600,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ const expected = ["- top-level", "- [warn] a warning"].join("\n");
+ expect(renderEasyView(records)).toBe(expected);
+ });
+
+ it("skips span-close records without a matching span-open", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "span-close",
+ spanId: "orphan",
+ name: "orphan",
+ timestamp: 1000,
+ durationMs: 100,
+ status: "ok",
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ {
+ kind: "log",
+ level: "info",
+ msg: "real",
+ timestamp: 2000,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ expect(renderEasyView(records)).toBe("- real");
+ });
+
+ it("renders span with ERR status", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "span-open",
+ spanId: "s1",
+ name: "failing-step",
+ timestamp: 1000,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ {
+ kind: "span-close",
+ spanId: "s1",
+ name: "failing-step",
+ timestamp: 1100,
+ durationMs: 100,
+ status: "error",
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ expect(renderEasyView(records)).toBe("- failing-step 100ms ERR");
+ });
+
+ it("renders body hint when body is present", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "span-open",
+ spanId: "s1",
+ name: "prompt",
+ timestamp: 1000,
+ extensionId: "ext",
+ turnId: "t1",
+ body: "x".repeat(2100),
+ },
+ {
+ kind: "span-close",
+ spanId: "s1",
+ name: "prompt",
+ timestamp: 1100,
+ durationMs: 100,
+ status: "ok",
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ expect(renderEasyView(records)).toBe("- prompt 100ms [body 2.1k]");
+ });
+
+ it("renders log record body hint", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "log",
+ level: "debug",
+ msg: "payload",
+ timestamp: 500,
+ extensionId: "ext",
+ turnId: "t1",
+ body: "abc",
+ },
+ ];
+ expect(renderEasyView(records)).toBe("- [debug] payload [body 3b]");
+ });
+
+ it("renders attributes on span (up to 3)", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "span-open",
+ spanId: "s1",
+ name: "provider.request",
+ timestamp: 1000,
+ extensionId: "ext",
+ turnId: "t1",
+ attributes: { model: "gpt-4", method: "POST", status: 200 },
+ },
+ {
+ kind: "span-close",
+ spanId: "s1",
+ name: "provider.request",
+ timestamp: 1500,
+ durationMs: 500,
+ status: "ok",
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ const output = renderEasyView(records);
+ expect(output).toContain("provider.request 500ms");
+ expect(output).toContain('model="gpt-4"');
+ expect(output).toContain('method="POST"');
+ expect(output).toContain("status=200");
+ });
+
+ it("renders mixed records in correct order", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "span-open",
+ spanId: "s1",
+ name: "step",
+ timestamp: 1000,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ {
+ kind: "log",
+ level: "info",
+ msg: "inside step",
+ timestamp: 1200,
+ extensionId: "ext",
+ turnId: "t1",
+ parentSpanId: "s1",
+ },
+ {
+ kind: "span-close",
+ spanId: "s1",
+ name: "step",
+ timestamp: 2800,
+ durationMs: 1800,
+ status: "ok",
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ {
+ kind: "log",
+ level: "info",
+ msg: "after step",
+ timestamp: 3000,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ const expected = ["- step 1.8s", " - inside step", "- after step"].join("\n");
+ expect(renderEasyView(records)).toBe(expected);
+ });
+
+ it("deterministic output for same input", () => {
+ const records: LogRecord[] = [
+ {
+ kind: "span-open",
+ spanId: "s1",
+ name: "step",
+ timestamp: 1000,
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ {
+ kind: "span-close",
+ spanId: "s1",
+ name: "step",
+ timestamp: 1800,
+ durationMs: 800,
+ status: "ok",
+ extensionId: "ext",
+ turnId: "t1",
+ },
+ ];
+ expect(renderEasyView(records)).toBe(renderEasyView(records));
+ });
+});
+
+describe("formatDuration", () => {
+ it("formats milliseconds under 1s", () => {
+ expect(formatDuration(42)).toBe("42ms");
+ expect(formatDuration(999)).toBe("999ms");
+ });
+
+ it("formats seconds", () => {
+ expect(formatDuration(1000)).toBe("1.0s");
+ expect(formatDuration(1500)).toBe("1.5s");
+ expect(formatDuration(59999)).toBe("60.0s");
+ });
+
+ it("formats minutes", () => {
+ expect(formatDuration(60000)).toBe("1m0s");
+ expect(formatDuration(90000)).toBe("1m30s");
+ expect(formatDuration(125000)).toBe("2m5s");
+ });
+});
diff --git a/packages/trace-store/src/easy-view.ts b/packages/trace-store/src/easy-view.ts
new file mode 100644
index 0000000..55477ee
--- /dev/null
+++ b/packages/trace-store/src/easy-view.ts
@@ -0,0 +1,205 @@
+import type { Attributes, LogRecord } from "@dispatch/kernel";
+
+interface SpanInfo {
+ name: string;
+ openTimestamp: number;
+ closeTimestamp?: number;
+ durationMs?: number;
+ status?: string;
+ body?: string;
+ attributes?: Attributes;
+ children: TimelineEntry[];
+}
+
+interface LogEntry {
+ record: LogRecord;
+}
+
+type TimelineEntry = { type: "span"; span: SpanInfo } | { type: "log"; entry: LogEntry };
+
+export function renderEasyView(records: readonly LogRecord[]): string {
+ if (records.length === 0) {
+ return "";
+ }
+
+ const sorted = [...records].sort((a, b) => a.timestamp - b.timestamp);
+
+ const spansById = new Map<string, SpanInfo>();
+ const roots: TimelineEntry[] = [];
+ const childrenByParent = new Map<string | undefined, TimelineEntry[]>();
+
+ for (const r of sorted) {
+ if (r.kind === "span-open") {
+ const span: SpanInfo = {
+ name: r.name,
+ openTimestamp: r.timestamp,
+ ...(r.body !== undefined && { body: r.body }),
+ ...(r.attributes !== undefined && { attributes: r.attributes }),
+ children: [],
+ };
+ spansById.set(r.spanId, span);
+ } else if (r.kind === "span-close") {
+ const span = spansById.get(r.spanId);
+ if (span !== undefined) {
+ span.closeTimestamp = r.timestamp;
+ span.durationMs = r.durationMs;
+ span.status = r.status;
+ if (r.body !== undefined) {
+ span.body = r.body;
+ }
+ if (r.attributes !== undefined) {
+ span.attributes = { ...span.attributes, ...r.attributes };
+ }
+ }
+ } else {
+ const entry: TimelineEntry = { type: "log", entry: { record: r } };
+ const parentId = r.parentSpanId;
+ let siblings = childrenByParent.get(parentId);
+ if (siblings === undefined) {
+ siblings = [];
+ childrenByParent.set(parentId, siblings);
+ }
+ siblings.push(entry);
+ }
+ }
+
+ for (const [spanId, span] of spansById) {
+ const entry: TimelineEntry = { type: "span", span };
+ const parentId = getSpanParentId(sorted, spanId);
+ let siblings = childrenByParent.get(parentId);
+ if (siblings === undefined) {
+ siblings = [];
+ childrenByParent.set(parentId, siblings);
+ }
+ siblings.push(entry);
+ }
+
+ for (const [parentId, children] of childrenByParent) {
+ if (parentId === undefined) {
+ roots.push(...children);
+ } else {
+ const parent = spansById.get(parentId);
+ if (parent !== undefined) {
+ parent.children.push(...children);
+ } else {
+ roots.push(...children);
+ }
+ }
+ }
+
+ sortByTimestamp(roots);
+ for (const entry of roots) {
+ if (entry.type === "span") {
+ sortByTimestamp(entry.span.children);
+ }
+ }
+
+ const lines: string[] = [];
+ for (const entry of roots) {
+ renderEntry(entry, 0, lines);
+ }
+ return lines.join("\n");
+}
+
+function sortByTimestamp(entries: TimelineEntry[]): void {
+ entries.sort((a, b) => {
+ const tsA = a.type === "span" ? a.span.openTimestamp : a.entry.record.timestamp;
+ const tsB = b.type === "span" ? b.span.openTimestamp : b.entry.record.timestamp;
+ return tsA - tsB;
+ });
+}
+
+function getSpanParentId(records: readonly LogRecord[], spanId: string): string | undefined {
+ for (const r of records) {
+ if (r.kind === "span-open" && r.spanId === spanId) {
+ return r.parentSpanId;
+ }
+ }
+ return undefined;
+}
+
+function renderEntry(entry: TimelineEntry, depth: number, lines: string[]): void {
+ const indent = " ".repeat(depth);
+
+ if (entry.type === "span") {
+ const span = entry.span;
+ const dur = span.durationMs !== undefined ? formatDuration(span.durationMs) : undefined;
+ const statusStr = span.status === "error" ? " ERR" : "";
+ const suffix = dur !== undefined ? ` ${dur}${statusStr}` : " (open)";
+
+ let detail = "";
+ if (span.attributes !== undefined) {
+ const keys = Object.keys(span.attributes);
+ if (keys.length > 0) {
+ const parts: string[] = [];
+ for (const k of keys.slice(0, 3)) {
+ const v = span.attributes[k];
+ if (v !== undefined) {
+ parts.push(`${k}=${formatAttrValue(v)}`);
+ }
+ }
+ detail = ` {${parts.join(", ")}}`;
+ }
+ }
+
+ const bodyHint = span.body !== undefined ? ` [body ${formatSize(span.body.length)}]` : "";
+
+ lines.push(`${indent}- ${span.name}${suffix}${detail}${bodyHint}`);
+
+ for (const child of span.children) {
+ renderEntry(child, depth + 1, lines);
+ }
+ } else {
+ const r = entry.entry.record;
+ if (r.kind === "log") {
+ const lvl = levelTag(r.level);
+ const bodyHint = r.body !== undefined ? ` [body ${formatSize(r.body.length)}]` : "";
+ lines.push(`${indent}- ${lvl}${r.msg}${bodyHint}`);
+ }
+ }
+}
+
+function levelTag(level: string): string {
+ if (level === "info") {
+ return "";
+ }
+ return `[${level}] `;
+}
+
+export function formatDuration(ms: number): string {
+ if (ms < 1000) {
+ return `${ms}ms`;
+ }
+ const s = ms / 1000;
+ if (s < 60) {
+ return `${s.toFixed(1)}s`;
+ }
+ const m = Math.floor(s / 60);
+ const rem = (s % 60).toFixed(0);
+ return `${m}m${rem}s`;
+}
+
+function formatSize(n: number): string {
+ if (n < 1024) {
+ return `${n}b`;
+ }
+ const kb = n / 1024;
+ if (kb < 1024) {
+ return `${kb.toFixed(1)}k`;
+ }
+ const mb = kb / 1024;
+ return `${mb.toFixed(1)}M`;
+}
+
+function formatAttrValue(value: string | number | boolean | null): string {
+ if (value === null) {
+ return "null";
+ }
+ if (typeof value === "string") {
+ if (value.length > 20) {
+ return `"${value.slice(0, 17)}..."`;
+ }
+ return `"${value}"`;
+ }
+ return String(value);
+}
diff --git a/packages/trace-store/src/index.ts b/packages/trace-store/src/index.ts
new file mode 100644
index 0000000..7341305
--- /dev/null
+++ b/packages/trace-store/src/index.ts
@@ -0,0 +1,3 @@
+export { formatDuration, renderEasyView } from "./easy-view.js";
+export type { TraceStore } from "./store.js";
+export { createTraceStore, stableId } from "./store.js";
diff --git a/packages/trace-store/src/store.test.ts b/packages/trace-store/src/store.test.ts
new file mode 100644
index 0000000..e380147
--- /dev/null
+++ b/packages/trace-store/src/store.test.ts
@@ -0,0 +1,224 @@
+import type { LogRecord } from "@dispatch/kernel";
+import { describe, expect, it } from "vitest";
+import { createTraceStore, stableId } from "./store.js";
+
+const logRecord: LogRecord = {
+ kind: "log",
+ level: "info",
+ msg: "hello",
+ timestamp: 1700000000000,
+ extensionId: "test-ext",
+ conversationId: "conv-1",
+ turnId: "turn-1",
+ spanId: "span-1",
+ attributes: { key: "value" },
+};
+
+const spanOpenRecord: LogRecord = {
+ kind: "span-open",
+ spanId: "span-2",
+ name: "step",
+ timestamp: 1700000000100,
+ extensionId: "test-ext",
+ conversationId: "conv-1",
+ turnId: "turn-1",
+ parentSpanId: "span-1",
+};
+
+const spanCloseRecord: LogRecord = {
+ kind: "span-close",
+ spanId: "span-2",
+ name: "step",
+ timestamp: 1700000000500,
+ durationMs: 400,
+ status: "ok",
+ extensionId: "test-ext",
+ conversationId: "conv-1",
+ turnId: "turn-1",
+ parentSpanId: "span-1",
+};
+
+const bodyRecord: LogRecord = {
+ kind: "span-open",
+ spanId: "span-3",
+ name: "prompt",
+ timestamp: 1700000000200,
+ extensionId: "test-ext",
+ conversationId: "conv-1",
+ turnId: "turn-1",
+ body: "the full prompt text",
+};
+
+const logRecordNoBody: LogRecord = {
+ kind: "log",
+ level: "debug",
+ msg: "no body here",
+ timestamp: 1700000000050,
+ extensionId: "ext-min",
+ turnId: "turn-1",
+};
+
+describe("stableId", () => {
+ it("produces a 16-char hex string", () => {
+ const id = stableId(logRecord);
+ expect(id).toMatch(/^[0-9a-f]{16}$/);
+ });
+
+ it("is deterministic for the same record", () => {
+ expect(stableId(logRecord)).toBe(stableId(logRecord));
+ });
+
+ it("produces different ids for different records", () => {
+ expect(stableId(logRecord)).not.toBe(stableId(spanOpenRecord));
+ });
+});
+
+describe("createTraceStore", () => {
+ function freshStore() {
+ return createTraceStore({ path: ":memory:" });
+ }
+
+ it("inserts and retrieves records ordered by timestamp", () => {
+ const store = freshStore();
+ store.insertRecords([logRecord, spanCloseRecord, spanOpenRecord]);
+ const result = store.getTurn("turn-1");
+ expect(result).toHaveLength(3);
+ expect(result[0]?.timestamp).toBe(1700000000000);
+ expect(result[1]?.timestamp).toBe(1700000000100);
+ expect(result[2]?.timestamp).toBe(1700000000500);
+ store.close();
+ });
+
+ it("reconstructs attributes from JSON", () => {
+ const store = freshStore();
+ store.insertRecords([logRecord]);
+ const result = store.getTurn("turn-1");
+ expect(result[0]?.kind).toBe("log");
+ if (result[0]?.kind === "log") {
+ expect(result[0].attributes).toEqual({ key: "value" });
+ }
+ store.close();
+ });
+
+ it("reconstructs links from JSON", () => {
+ const store = freshStore();
+ const withLinks: LogRecord = {
+ kind: "span-open",
+ spanId: "span-link",
+ name: "linked",
+ timestamp: 1700000000999,
+ extensionId: "ext",
+ turnId: "turn-1",
+ links: [{ spanId: "other-span", turnId: "turn-0", reason: "caused" }],
+ };
+ store.insertRecords([withLinks]);
+ const result = store.getTurn("turn-1");
+ expect(result[0]?.kind).toBe("span-open");
+ if (result[0]?.kind === "span-open") {
+ expect(result[0].links).toEqual([
+ { spanId: "other-span", turnId: "turn-0", reason: "caused" },
+ ]);
+ }
+ store.close();
+ });
+
+ it("returns empty array for unknown turnId", () => {
+ const store = freshStore();
+ store.insertRecords([logRecord]);
+ expect(store.getTurn("nonexistent")).toEqual([]);
+ store.close();
+ });
+
+ it("getBody returns body for body-bearing records", () => {
+ const store = freshStore();
+ store.insertRecords([bodyRecord]);
+ const id = stableId(bodyRecord);
+ expect(store.getBody(id)).toBe("the full prompt text");
+ store.close();
+ });
+
+ it("getBody returns undefined for records without body", () => {
+ const store = freshStore();
+ store.insertRecords([logRecordNoBody]);
+ const id = stableId(logRecordNoBody);
+ expect(store.getBody(id)).toBeUndefined();
+ store.close();
+ });
+
+ it("bodies table only holds body-bearing records", () => {
+ const store = freshStore();
+ store.insertRecords([logRecord, bodyRecord, logRecordNoBody]);
+ const bodyId = stableId(bodyRecord);
+ expect(store.getBody(bodyId)).toBe("the full prompt text");
+
+ const logId = stableId(logRecord);
+ expect(store.getBody(logId)).toBeUndefined();
+
+ const noBodyId = stableId(logRecordNoBody);
+ expect(store.getBody(noBodyId)).toBeUndefined();
+ store.close();
+ });
+
+ it("is idempotent — re-inserting the same records produces no duplicates", () => {
+ const store = freshStore();
+ store.insertRecords([logRecord, spanOpenRecord, spanCloseRecord, bodyRecord]);
+ store.insertRecords([logRecord, spanOpenRecord, spanCloseRecord, bodyRecord]);
+ const result = store.getTurn("turn-1");
+ expect(result).toHaveLength(4);
+ store.close();
+ });
+
+ it("handles span-close record with attributes and links round-trip", () => {
+ const store = freshStore();
+ const closeWithMeta: LogRecord = {
+ kind: "span-close",
+ spanId: "span-m",
+ name: "step",
+ timestamp: 1700000001000,
+ durationMs: 250,
+ status: "error",
+ extensionId: "ext",
+ turnId: "turn-1",
+ attributes: { httpStatus: 500 },
+ links: [{ spanId: "upstream" }],
+ };
+ store.insertRecords([closeWithMeta]);
+ const result = store.getTurn("turn-1");
+ expect(result[0]?.kind).toBe("span-close");
+ if (result[0]?.kind === "span-close") {
+ expect(result[0].durationMs).toBe(250);
+ expect(result[0].status).toBe("error");
+ expect(result[0].attributes).toEqual({ httpStatus: 500 });
+ expect(result[0].links).toEqual([{ spanId: "upstream" }]);
+ }
+ store.close();
+ });
+
+ it("easyView delegates to renderEasyView", () => {
+ const store = freshStore();
+ store.insertRecords([spanOpenRecord]);
+ const output = store.easyView("turn-1");
+ expect(output).toContain("step (open)");
+ store.close();
+ });
+
+ it("persists to a file path", () => {
+ const tmpPath = `/tmp/trace-store-test-${Date.now()}.db`;
+ const store = createTraceStore({ path: tmpPath });
+ store.insertRecords([logRecord]);
+ store.close();
+
+ const store2 = createTraceStore({ path: tmpPath });
+ const result = store2.getTurn("turn-1");
+ expect(result).toHaveLength(1);
+ expect(result[0]?.kind).toBe("log");
+ store2.close();
+
+ const { unlinkSync } = require("node:fs");
+ try {
+ unlinkSync(tmpPath);
+ } catch {
+ // ignore cleanup error
+ }
+ });
+});
diff --git a/packages/trace-store/src/store.ts b/packages/trace-store/src/store.ts
new file mode 100644
index 0000000..53b7390
--- /dev/null
+++ b/packages/trace-store/src/store.ts
@@ -0,0 +1,254 @@
+import { Database } from "bun:sqlite";
+import type { Attributes, LogRecord, SpanLink } from "@dispatch/kernel";
+import { renderEasyView } from "./easy-view.js";
+
+export interface TraceStore {
+ insertRecords(records: readonly LogRecord[]): void;
+ getTurn(turnId: string): LogRecord[];
+ getBody(recordId: string): string | undefined;
+ easyView(turnId: string): string;
+ close(): void;
+}
+
+export function createTraceStore(opts: { path: string }): TraceStore {
+ const db = new Database(opts.path);
+ db.run("PRAGMA journal_mode = WAL");
+ schema(db);
+ return {
+ insertRecords(records) {
+ insertRecords(db, records);
+ },
+ getTurn(turnId) {
+ return getTurn(db, turnId);
+ },
+ getBody(recordId) {
+ return getBody(db, recordId);
+ },
+ easyView(turnId) {
+ return renderEasyView(getTurn(db, turnId));
+ },
+ close() {
+ db.close();
+ },
+ };
+}
+
+function schema(db: Database): void {
+ db.run(`
+ CREATE TABLE IF NOT EXISTS records (
+ id TEXT PRIMARY KEY,
+ kind TEXT NOT NULL,
+ level TEXT,
+ msg TEXT,
+ name TEXT,
+ spanId TEXT,
+ parentSpanId TEXT,
+ conversationId TEXT,
+ turnId TEXT,
+ extensionId TEXT NOT NULL,
+ timestamp INTEGER NOT NULL,
+ durationMs INTEGER,
+ status TEXT,
+ attributes TEXT,
+ links TEXT
+ )
+ `);
+ db.run("CREATE INDEX IF NOT EXISTS idx_records_turnId ON records(turnId)");
+ db.run("CREATE INDEX IF NOT EXISTS idx_records_conversationId ON records(conversationId)");
+ db.run("CREATE INDEX IF NOT EXISTS idx_records_spanId ON records(spanId)");
+ db.run("CREATE INDEX IF NOT EXISTS idx_records_kind ON records(kind)");
+ db.run("CREATE INDEX IF NOT EXISTS idx_records_timestamp ON records(timestamp)");
+
+ db.run(`
+ CREATE TABLE IF NOT EXISTS bodies (
+ recordId TEXT PRIMARY KEY REFERENCES records(id),
+ body TEXT NOT NULL
+ )
+ `);
+}
+
+function insertRecords(db: Database, records: readonly LogRecord[]): void {
+ const recStmt = db.prepare(`
+ INSERT OR IGNORE INTO records
+ (id, kind, level, msg, name, spanId, parentSpanId,
+ conversationId, turnId, extensionId, timestamp,
+ durationMs, status, attributes, links)
+ VALUES
+ (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ `);
+ const bodyStmt = db.prepare("INSERT OR IGNORE INTO bodies (recordId, body) VALUES (?, ?)");
+
+ const txn = db.transaction(() => {
+ for (const r of records) {
+ const id = stableId(r);
+ const kind = r.kind;
+ let level: string | null = null;
+ let msg: string | null = null;
+ let name: string | null = null;
+ let spanId: string | null = null;
+ let parentSpanId: string | null = null;
+ let durationMs: number | null = null;
+ let status: string | null = null;
+ let links: string | null = null;
+
+ if (r.kind === "log") {
+ level = r.level;
+ msg = r.msg;
+ spanId = r.spanId ?? null;
+ parentSpanId = r.parentSpanId ?? null;
+ } else if (r.kind === "span-open") {
+ name = r.name;
+ spanId = r.spanId;
+ parentSpanId = r.parentSpanId ?? null;
+ if (r.links !== undefined) {
+ links = JSON.stringify(r.links);
+ }
+ } else {
+ name = r.name;
+ spanId = r.spanId;
+ parentSpanId = r.parentSpanId ?? null;
+ durationMs = r.durationMs;
+ status = r.status;
+ if (r.links !== undefined) {
+ links = JSON.stringify(r.links);
+ }
+ }
+
+ const attributes: string | null =
+ r.attributes !== undefined ? JSON.stringify(r.attributes) : null;
+
+ recStmt.run(
+ id,
+ kind,
+ level,
+ msg,
+ name,
+ spanId,
+ parentSpanId,
+ r.conversationId ?? null,
+ r.turnId ?? null,
+ r.extensionId,
+ r.timestamp,
+ durationMs,
+ status,
+ attributes,
+ links,
+ );
+
+ if (r.body !== undefined) {
+ bodyStmt.run(id, r.body);
+ }
+ }
+ });
+ txn();
+}
+
+interface RecordRow {
+ id: string;
+ kind: string;
+ level: string | null;
+ msg: string | null;
+ name: string | null;
+ spanId: string | null;
+ parentSpanId: string | null;
+ conversationId: string | null;
+ turnId: string | null;
+ extensionId: string;
+ timestamp: number;
+ durationMs: number | null;
+ status: string | null;
+ attributes: string | null;
+ links: string | null;
+}
+
+function getTurn(db: Database, turnId: string): LogRecord[] {
+ const rows = db
+ .query("SELECT * FROM records WHERE turnId = ? ORDER BY timestamp ASC, rowid ASC")
+ .all(turnId) as RecordRow[];
+ return rows.map(rowToRecord);
+}
+
+function getBody(db: Database, recordId: string): string | undefined {
+ const row = db.query("SELECT body FROM bodies WHERE recordId = ?").get(recordId) as {
+ body: string;
+ } | null;
+ return row?.body;
+}
+
+function rowToRecord(row: RecordRow): LogRecord {
+ const attributes: Attributes | undefined =
+ row.attributes !== null ? JSON.parse(row.attributes) : undefined;
+ const links: SpanLink[] | undefined = row.links !== null ? JSON.parse(row.links) : undefined;
+
+ if (row.kind === "log") {
+ const record: LogRecord = {
+ kind: "log",
+ level: row.level as "debug" | "info" | "warn" | "error",
+ msg: row.msg ?? "",
+ timestamp: row.timestamp,
+ extensionId: row.extensionId,
+ ...(row.conversationId !== null && { conversationId: row.conversationId }),
+ ...(row.turnId !== null && { turnId: row.turnId }),
+ ...(row.spanId !== null && { spanId: row.spanId }),
+ ...(row.parentSpanId !== null && { parentSpanId: row.parentSpanId }),
+ ...(attributes !== undefined && { attributes }),
+ };
+ return record;
+ }
+
+ if (row.kind === "span-open") {
+ const record: LogRecord = {
+ kind: "span-open",
+ spanId: row.spanId ?? "",
+ name: row.name ?? "",
+ timestamp: row.timestamp,
+ extensionId: row.extensionId,
+ ...(row.conversationId !== null && { conversationId: row.conversationId }),
+ ...(row.turnId !== null && { turnId: row.turnId }),
+ ...(row.parentSpanId !== null && { parentSpanId: row.parentSpanId }),
+ ...(attributes !== undefined && { attributes }),
+ ...(links !== undefined && { links }),
+ };
+ return record;
+ }
+
+ const record: LogRecord = {
+ kind: "span-close",
+ spanId: row.spanId ?? "",
+ name: row.name ?? "",
+ timestamp: row.timestamp,
+ durationMs: row.durationMs ?? 0,
+ status: (row.status as "ok" | "error") ?? "ok",
+ extensionId: row.extensionId,
+ ...(row.conversationId !== null && { conversationId: row.conversationId }),
+ ...(row.turnId !== null && { turnId: row.turnId }),
+ ...(row.parentSpanId !== null && { parentSpanId: row.parentSpanId }),
+ ...(attributes !== undefined && { attributes }),
+ ...(links !== undefined && { links }),
+ };
+ return record;
+}
+
+function toCanonicalJson(value: unknown): string {
+ if (value === null || typeof value !== "object") {
+ return JSON.stringify(value);
+ }
+ if (Array.isArray(value)) {
+ return `[${value.map(toCanonicalJson).join(",")}]`;
+ }
+ const obj = value as Record<string, unknown>;
+ const keys = Object.keys(obj).sort();
+ const entries = keys.map((k) => `${JSON.stringify(k)}:${toCanonicalJson(obj[k])}`);
+ return `{${entries.join(",")}}`;
+}
+
+export function stableId(record: LogRecord): string {
+ const json = toCanonicalJson(record);
+ let hash = 0xcbf29ce484222325n;
+ const prime = 0x100000001b3n;
+ for (let i = 0; i < json.length; i++) {
+ hash ^= BigInt(json.charCodeAt(i));
+ hash = (hash * prime) & 0xffffffffffffffffn;
+ }
+ return hash.toString(16).padStart(16, "0");
+}
diff --git a/packages/trace-store/tsconfig.json b/packages/trace-store/tsconfig.json
new file mode 100644
index 0000000..ff99a43
--- /dev/null
+++ b/packages/trace-store/tsconfig.json
@@ -0,0 +1,6 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": { "rootDir": "src", "outDir": "dist", "composite": true },
+ "include": ["src/**/*.ts"],
+ "references": [{ "path": "../kernel" }]
+}
diff --git a/tasks.md b/tasks.md
index 61daa95..2294e7e 100644
--- a/tasks.md
+++ b/tasks.md
@@ -206,11 +206,27 @@ per-extension self-redaction (no shared helper — isolation over DRY).
present, correlated (shared turnId), `request.body` no longer in attributes, key leak 0.
Summons: prompts/phase-a3-{kernel-body-channel,provider-body}.md.
+### Phase B — collector + trace store ✅ DONE + proven (first slice)
+- [x] **trace-store** (`packages/trace-store/`, `bun:sqlite`): records+bodies schema
+ (thin/fat split), idempotent `insertRecords` (FNV-1a id + `INSERT OR IGNORE`),
+ `getTurn`/`getBody`, pure `renderEasyView` (D8 timeline skeleton), `trace` CLI. 30 tests.
+- [x] **observability-collector** (`packages/observability-collector/`): out-of-process
+ bin — tail journal → `splitLines`/`drainOnce` → `insertRecords`; offset sidecar;
+ at-least-once + idempotent; fail-safe; clean SIGINT/SIGTERM drain. 21 tests.
+- [x] **Build-config wiring** (orchestrator): root tsconfig refs; both excluded from
+ vitest + added to `test:bun` (`bun:sqlite`); `bun install`.
+- typecheck clean, **345 tests** (273 vitest + 72 bun), biome 0/0. **Pipeline proven:**
+ app → journal → collector → SQLite → `trace <turnId>` easy-view.
+ Summons: prompts/phase-b-{trace-store,observability-collector}.md.
+
### Next (observability)
-- **Phase B:** out-of-process collector → SQLite store + query (§11).
-- **Record/replay test fixtures** (goal): turn captured verbatim provider.request/
- response traces into hermetic `stream.test.ts` fixtures (mock `fetch`, replay real
- flash) for regression + deterministic repro. D5; §7. Complements contract-fakes.
+- **Span nesting fix (kernel run-turn):** spans are currently flat (all `parent=ROOT`);
+ nest `step`←`turn` and `prompt`/`provider.request`←`step` (pass the step span's logger
+ into `provider.stream`) so the trace is a tree. (`renderEasyView` already nests once
+ parents exist.)
+- **host-bin supervision** (deferred): spawn-first / drain-last / restart the collector.
+- **Record/replay test fixtures** (goal): captured verbatim provider.request/response
+ traces → hermetic `stream.test.ts` fixtures (mock `fetch`, replay real flash). D5; §7.
Summons: prompts/phase-a-{kernel-logging,journal-sink}.md;
reports/phase-a-{kernel-logging,journal-sink}.md.
diff --git a/tsconfig.json b/tsconfig.json
index 0f5f80f..a824844 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -10,6 +10,8 @@
{ "path": "./packages/transport-http" },
{ "path": "./packages/tool-read-file" },
{ "path": "./packages/journal-sink" },
+ { "path": "./packages/trace-store" },
+ { "path": "./packages/observability-collector" },
{ "path": "./packages/host-bin" }
]
}
diff --git a/vitest.config.ts b/vitest.config.ts
index f682bbf..e04946d 100644
--- a/vitest.config.ts
+++ b/vitest.config.ts
@@ -5,6 +5,11 @@ export default defineConfig({
// Packages whose code imports Bun-only modules (e.g. `bun:sqlite`) can't run
// under Vite's Node transform — they test via `bun test` (see `test:bun`).
// Everything else runs here under vitest.
- projects: ["packages/*", "!packages/storage-sqlite"],
+ projects: [
+ "packages/*",
+ "!packages/storage-sqlite",
+ "!packages/trace-store",
+ "!packages/observability-collector",
+ ],
},
});