diff options
| author | Adam Malczewski <[email protected]> | 2026-06-04 23:22:03 +0900 |
|---|---|---|
| committer | Adam Malczewski <[email protected]> | 2026-06-04 23:22:03 +0900 |
| commit | 9b611d614d123462e50492d78202dae696b99aa2 (patch) | |
| tree | 7e0c85b60dd0becddbb04c29fdf14855c10c090f | |
| parent | 974ce6f46c25a522a42c6bd04fd62ce2d031aad5 (diff) | |
| download | dispatch-9b611d614d123462e50492d78202dae696b99aa2.tar.gz dispatch-9b611d614d123462e50492d78202dae696b99aa2.zip | |
feat(core-ext): storage-sqlite, auth-apikey, provider-openai-compat
- storage-sqlite: bun:sqlite StorageNamespace backend + migrations (21 bun tests)
- auth-apikey: pure resolver from env → ApiKeyCredentials (4 tests)
- provider-openai-compat: OpenAI-compatible SSE stream → ProviderEvents
- orchestrator fixes: provider imports (@dispatch/kernel), missing dep,
exactOptionalPropertyTypes (omit-when-undefined), root tsconfig refs
- vitest excludes storage-sqlite (bun:sqlite); test:bun runs it under bun
30 files changed, 1620 insertions, 21 deletions
@@ -11,10 +11,31 @@ "vitest": "^3.0.0", }, }, + "packages/auth-apikey": { + "name": "@dispatch/auth-apikey", + "version": "0.0.0", + "dependencies": { + "@dispatch/kernel": "workspace:*", + }, + }, "packages/kernel": { "name": "@dispatch/kernel", "version": "0.0.0", }, + "packages/provider-openai-compat": { + "name": "@dispatch/provider-openai-compat", + "version": "0.0.0", + "dependencies": { + "@dispatch/kernel": "workspace:*", + }, + }, + "packages/storage-sqlite": { + "name": "@dispatch/storage-sqlite", + "version": "0.0.0", + "dependencies": { + "@dispatch/kernel": "workspace:*", + }, + }, }, "packages": { "@biomejs/biome": ["@biomejs/[email protected]", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.16", "@biomejs/cli-darwin-x64": "2.4.16", "@biomejs/cli-linux-arm64": "2.4.16", "@biomejs/cli-linux-arm64-musl": "2.4.16", "@biomejs/cli-linux-x64": "2.4.16", "@biomejs/cli-linux-x64-musl": "2.4.16", "@biomejs/cli-win32-arm64": "2.4.16", "@biomejs/cli-win32-x64": "2.4.16" }, "bin": { "biome": "bin/biome" } }, "sha512-x9ajFh1zChVybCiM3TN6OD4phAqLgtPZjFrZF+aTMYCPjwBO+k529TX7PPsAqtGNLeV4UgzwQnowEgS7bGmzcA=="], @@ -35,8 +56,14 @@ "@biomejs/cli-win32-x64": ["@biomejs/[email protected]", "", { "os": "win32", "cpu": "x64" }, "sha512-Kp85jgoBHa05gix6UIRjfCDiUV3w/8VIdZ247VyyO2gEjaw12WEVhdIjlxp/AMzXxqxQwbxNTDVZ3Mwd2RG5rw=="], + "@dispatch/auth-apikey": ["@dispatch/auth-apikey@workspace:packages/auth-apikey"], + "@dispatch/kernel": ["@dispatch/kernel@workspace:packages/kernel"], + "@dispatch/provider-openai-compat": ["@dispatch/provider-openai-compat@workspace:packages/provider-openai-compat"], + + "@dispatch/storage-sqlite": ["@dispatch/storage-sqlite@workspace:packages/storage-sqlite"], + "@esbuild/aix-ppc64": ["@esbuild/[email protected]", "", { "os": "aix", "cpu": "ppc64" }, "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg=="], "@esbuild/android-arm": ["@esbuild/[email protected]", "", { "os": "android", "cpu": "arm" }, "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ=="], diff --git a/package.json b/package.json index dcdf705..5472946 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,9 @@ "check:fix": "biome check --write .", "test": "vitest run", "test:watch": "vitest", - "typecheck": "tsc -b --pretty" + "typecheck": "tsc -b --pretty", + "test:bun": "bun test packages/storage-sqlite/src", + "test:all": "bun run test && bun run test:bun" }, "devDependencies": { "@biomejs/biome": "^2.4.15", diff --git a/packages/auth-apikey/package.json b/packages/auth-apikey/package.json new file mode 100644 index 0000000..74ee65f --- /dev/null +++ b/packages/auth-apikey/package.json @@ -0,0 +1,11 @@ +{ + "name": "@dispatch/auth-apikey", + "version": "0.0.0", + "type": "module", + "private": true, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "dependencies": { + "@dispatch/kernel": "workspace:*" + } +} diff --git a/packages/auth-apikey/src/extension.ts b/packages/auth-apikey/src/extension.ts new file mode 100644 index 0000000..05d6462 --- /dev/null +++ b/packages/auth-apikey/src/extension.ts @@ -0,0 +1,22 @@ +import type { AuthContract, Extension } from "@dispatch/kernel"; +import { resolveApiKeyCredentials } from "./resolver.js"; + +export const apikeyAuth: AuthContract = { + id: "apikey", + resolve: async () => + resolveApiKeyCredentials(process.env as Readonly<Record<string, string | undefined>>), +}; + +export const extension: Extension = { + manifest: { + id: "auth-apikey", + name: "API Key Auth", + version: "0.0.0", + apiVersion: "^0.1.0", + trust: "bundled", + contributes: { auth: ["apikey"] }, + }, + activate(host) { + host.defineAuth(apikeyAuth); + }, +}; diff --git a/packages/auth-apikey/src/index.ts b/packages/auth-apikey/src/index.ts new file mode 100644 index 0000000..450ba30 --- /dev/null +++ b/packages/auth-apikey/src/index.ts @@ -0,0 +1,2 @@ +export { apikeyAuth, extension } from "./extension.js"; +export { resolveApiKeyCredentials } from "./resolver.js"; diff --git a/packages/auth-apikey/src/resolver.test.ts b/packages/auth-apikey/src/resolver.test.ts new file mode 100644 index 0000000..d6c1f85 --- /dev/null +++ b/packages/auth-apikey/src/resolver.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { resolveApiKeyCredentials } from "./resolver.js"; + +describe("resolveApiKeyCredentials", () => { + it("resolves key with default baseURL from minimal env", () => { + const env = { DISPATCH_API_KEY: "sk-test-123" }; + const result = resolveApiKeyCredentials(env); + expect(result).toEqual({ + type: "api-key", + apiKey: "sk-test-123", + baseURL: "https://opencode.ai/zen/go/v1", + }); + }); + + it("honors an explicit DISPATCH_BASE_URL", () => { + const env = { + DISPATCH_API_KEY: "sk-test-456", + DISPATCH_BASE_URL: "https://custom.example.com/v2", + }; + const result = resolveApiKeyCredentials(env); + expect(result).toEqual({ + type: "api-key", + apiKey: "sk-test-456", + baseURL: "https://custom.example.com/v2", + }); + }); + + it("throws a clear error when DISPATCH_API_KEY is absent", () => { + const env = {}; + expect(() => resolveApiKeyCredentials(env)).toThrow("DISPATCH_API_KEY"); + }); + + it("throws when DISPATCH_API_KEY is empty string", () => { + const env = { DISPATCH_API_KEY: "" }; + expect(() => resolveApiKeyCredentials(env)).toThrow("DISPATCH_API_KEY"); + }); +}); diff --git a/packages/auth-apikey/src/resolver.ts b/packages/auth-apikey/src/resolver.ts new file mode 100644 index 0000000..2e63a1c --- /dev/null +++ b/packages/auth-apikey/src/resolver.ts @@ -0,0 +1,14 @@ +import type { ApiKeyCredentials } from "@dispatch/kernel"; + +const DEFAULT_BASE_URL = "https://opencode.ai/zen/go/v1"; + +export function resolveApiKeyCredentials( + env: Readonly<Record<string, string | undefined>>, +): ApiKeyCredentials { + const apiKey = env.DISPATCH_API_KEY; + if (!apiKey) { + throw new Error("DISPATCH_API_KEY is not set. Set it in your environment or .env file."); + } + const baseURL = env.DISPATCH_BASE_URL ?? DEFAULT_BASE_URL; + return { type: "api-key", apiKey, baseURL }; +} diff --git a/packages/auth-apikey/tsconfig.json b/packages/auth-apikey/tsconfig.json new file mode 100644 index 0000000..ff99a43 --- /dev/null +++ b/packages/auth-apikey/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "rootDir": "src", "outDir": "dist", "composite": true }, + "include": ["src/**/*.ts"], + "references": [{ "path": "../kernel" }] +} diff --git a/packages/provider-openai-compat/package.json b/packages/provider-openai-compat/package.json new file mode 100644 index 0000000..df15fd3 --- /dev/null +++ b/packages/provider-openai-compat/package.json @@ -0,0 +1,11 @@ +{ + "name": "@dispatch/provider-openai-compat", + "version": "0.0.0", + "type": "module", + "private": true, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "dependencies": { + "@dispatch/kernel": "workspace:*" + } +} diff --git a/packages/provider-openai-compat/src/convert-messages.test.ts b/packages/provider-openai-compat/src/convert-messages.test.ts new file mode 100644 index 0000000..51513ea --- /dev/null +++ b/packages/provider-openai-compat/src/convert-messages.test.ts @@ -0,0 +1,259 @@ +import type { ChatMessage } from "@dispatch/kernel"; +import { describe, expect, it } from "vitest"; +import { convertMessages } from "./convert-messages.js"; + +describe("convertMessages", () => { + it("converts a system message with text chunks", () => { + const messages: ChatMessage[] = [ + { + role: "system", + chunks: [ + { type: "system", text: "You are a helpful assistant." }, + { type: "text", text: " Additional context." }, + ], + }, + ]; + + const result = convertMessages(messages); + expect(result).toEqual([ + { role: "system", content: "You are a helpful assistant. Additional context." }, + ]); + }); + + it("converts a user message with text chunks", () => { + const messages: ChatMessage[] = [ + { + role: "user", + chunks: [ + { type: "text", text: "Hello, " }, + { type: "text", text: "world!" }, + ], + }, + ]; + + const result = convertMessages(messages); + expect(result).toEqual([{ role: "user", content: "Hello, world!" }]); + }); + + it("converts an assistant message with text only", () => { + const messages: ChatMessage[] = [ + { + role: "assistant", + chunks: [ + { type: "text", text: "I can help " }, + { type: "text", text: "with that." }, + ], + }, + ]; + + const result = convertMessages(messages); + expect(result).toEqual([{ role: "assistant", content: "I can help with that." }]); + }); + + it("converts an assistant message with tool calls", () => { + const messages: ChatMessage[] = [ + { + role: "assistant", + chunks: [ + { type: "text", text: "Let me check that." }, + { + type: "tool-call", + toolCallId: "call_1", + toolName: "read_file", + input: { path: "/src/main.ts" }, + }, + ], + }, + ]; + + const result = convertMessages(messages); + expect(result).toEqual([ + { + role: "assistant", + content: "Let me check that.", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { + name: "read_file", + arguments: JSON.stringify({ path: "/src/main.ts" }), + }, + }, + ], + }, + ]); + }); + + it("converts an assistant message with tool calls but no text", () => { + const messages: ChatMessage[] = [ + { + role: "assistant", + chunks: [ + { + type: "tool-call", + toolCallId: "call_2", + toolName: "run_shell", + input: { command: "ls" }, + }, + ], + }, + ]; + + const result = convertMessages(messages); + expect(result).toEqual([ + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_2", + type: "function", + function: { + name: "run_shell", + arguments: JSON.stringify({ command: "ls" }), + }, + }, + ], + }, + ]); + }); + + it("converts tool result messages", () => { + const messages: ChatMessage[] = [ + { + role: "tool", + chunks: [ + { + type: "tool-result", + toolCallId: "call_1", + toolName: "read_file", + content: "file contents here", + isError: false, + }, + ], + }, + ]; + + const result = convertMessages(messages); + expect(result).toEqual([ + { + role: "tool", + content: "file contents here", + tool_call_id: "call_1", + }, + ]); + }); + + it("converts a full multi-turn history with tool round-trip", () => { + const messages: ChatMessage[] = [ + { + role: "system", + chunks: [{ type: "system", text: "You are helpful." }], + }, + { + role: "user", + chunks: [{ type: "text", text: "Read main.ts" }], + }, + { + role: "assistant", + chunks: [ + { type: "text", text: "Sure." }, + { + type: "tool-call", + toolCallId: "call_1", + toolName: "read_file", + input: { path: "main.ts" }, + }, + ], + }, + { + role: "tool", + chunks: [ + { + type: "tool-result", + toolCallId: "call_1", + toolName: "read_file", + content: "console.log('hello')", + isError: false, + }, + ], + }, + { + role: "assistant", + chunks: [{ type: "text", text: "The file logs hello." }], + }, + ]; + + const result = convertMessages(messages); + expect(result).toEqual([ + { role: "system", content: "You are helpful." }, + { role: "user", content: "Read main.ts" }, + { + role: "assistant", + content: "Sure.", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { + name: "read_file", + arguments: JSON.stringify({ path: "main.ts" }), + }, + }, + ], + }, + { + role: "tool", + content: "console.log('hello')", + tool_call_id: "call_1", + }, + { role: "assistant", content: "The file logs hello." }, + ]); + }); + + it("handles multiple tool results in one tool message", () => { + const messages: ChatMessage[] = [ + { + role: "tool", + chunks: [ + { + type: "tool-result", + toolCallId: "call_1", + toolName: "read_file", + content: "file1", + isError: false, + }, + { + type: "tool-result", + toolCallId: "call_2", + toolName: "read_file", + content: "file2", + isError: false, + }, + ], + }, + ]; + + const result = convertMessages(messages); + expect(result).toEqual([ + { role: "tool", content: "file1", tool_call_id: "call_1" }, + { role: "tool", content: "file2", tool_call_id: "call_2" }, + ]); + }); + + it("includes thinking chunks in assistant content", () => { + const messages: ChatMessage[] = [ + { + role: "assistant", + chunks: [ + { type: "thinking", text: "Let me think..." }, + { type: "text", text: "Here is my answer." }, + ], + }, + ]; + + const result = convertMessages(messages); + expect(result).toEqual([{ role: "assistant", content: "Let me think...Here is my answer." }]); + }); +}); diff --git a/packages/provider-openai-compat/src/convert-messages.ts b/packages/provider-openai-compat/src/convert-messages.ts new file mode 100644 index 0000000..786a70d --- /dev/null +++ b/packages/provider-openai-compat/src/convert-messages.ts @@ -0,0 +1,99 @@ +import type { ChatMessage, Chunk } from "@dispatch/kernel"; + +export interface OpenAIMessage { + readonly role: "system" | "user" | "assistant" | "tool"; + readonly content: string | null; + readonly tool_calls?: readonly OpenAIToolCall[]; + readonly tool_call_id?: string; +} + +export interface OpenAIToolCall { + readonly id: string; + readonly type: "function"; + readonly function: { readonly name: string; readonly arguments: string }; +} + +export function convertMessages(messages: readonly ChatMessage[]): OpenAIMessage[] { + const result: OpenAIMessage[] = []; + for (const msg of messages) { + const converted = convertMessage(msg); + for (const m of converted) { + result.push(m); + } + } + return result; +} + +function convertMessage(msg: ChatMessage): OpenAIMessage[] { + switch (msg.role) { + case "system": + return [convertSystemMessage(msg)]; + case "user": + return [convertUserMessage(msg)]; + case "assistant": + return [convertAssistantMessage(msg)]; + case "tool": + return convertToolResultMessages(msg); + } +} + +function convertSystemMessage(msg: ChatMessage): OpenAIMessage { + const text = msg.chunks + .filter( + (c): c is Extract<Chunk, { type: "text" | "system" }> => + c.type === "text" || c.type === "system", + ) + .map((c) => c.text) + .join(""); + return { role: "system", content: text }; +} + +function convertUserMessage(msg: ChatMessage): OpenAIMessage { + const text = msg.chunks + .filter((c): c is Extract<Chunk, { type: "text" }> => c.type === "text") + .map((c) => c.text) + .join(""); + return { role: "user", content: text }; +} + +function convertAssistantMessage(msg: ChatMessage): OpenAIMessage { + const textChunks = msg.chunks.filter( + (c): c is Extract<Chunk, { type: "text" | "thinking" }> => + c.type === "text" || c.type === "thinking", + ); + const content = textChunks.map((c) => c.text).join(""); + + const toolCalls = msg.chunks + .filter((c): c is Extract<Chunk, { type: "tool-call" }> => c.type === "tool-call") + .map( + (c): OpenAIToolCall => ({ + id: c.toolCallId, + type: "function", + function: { + name: c.toolName, + arguments: typeof c.input === "string" ? c.input : JSON.stringify(c.input), + }, + }), + ); + + if (toolCalls.length > 0) { + return { + role: "assistant", + content: content || null, + tool_calls: toolCalls, + }; + } + return { role: "assistant", content }; +} + +function convertToolResultMessages(msg: ChatMessage): OpenAIMessage[] { + return msg.chunks + .filter((c): c is Extract<Chunk, { type: "tool-result" }> => c.type === "tool-result") + .map( + (c): OpenAIMessage => ({ + role: "tool", + content: c.content, + tool_call_id: c.toolCallId, + }), + ); +} diff --git a/packages/provider-openai-compat/src/convert-tools.test.ts b/packages/provider-openai-compat/src/convert-tools.test.ts new file mode 100644 index 0000000..d739652 --- /dev/null +++ b/packages/provider-openai-compat/src/convert-tools.test.ts @@ -0,0 +1,106 @@ +import type { ToolContract } from "@dispatch/kernel"; +import { describe, expect, it } from "vitest"; +import { convertTools } from "./convert-tools.js"; + +describe("convertTools", () => { + it("converts a single tool to OpenAI function format", () => { + const tools: ToolContract[] = [ + { + name: "read_file", + description: "Read a file from disk", + parameters: { + type: "object", + properties: { + path: { type: "string", description: "File path" }, + }, + required: ["path"], + additionalProperties: false, + }, + execute: async () => ({ content: "" }), + }, + ]; + + const result = convertTools(tools); + expect(result).toEqual([ + { + type: "function", + function: { + name: "read_file", + description: "Read a file from disk", + parameters: { + type: "object", + properties: { + path: { type: "string", description: "File path" }, + }, + required: ["path"], + additionalProperties: false, + }, + }, + }, + ]); + }); + + it("converts multiple tools", () => { + const tools: ToolContract[] = [ + { + name: "read_file", + description: "Read a file", + parameters: { type: "object" }, + execute: async () => ({ content: "" }), + }, + { + name: "run_shell", + description: "Run a shell command", + parameters: { + type: "object", + properties: { + command: { type: "string", description: "The command" }, + }, + required: ["command"], + }, + execute: async () => ({ content: "" }), + }, + ]; + + const result = convertTools(tools); + expect(result).toHaveLength(2); + expect(result[0]?.function.name).toBe("read_file"); + expect(result[1]?.function.name).toBe("run_shell"); + }); + + it("returns empty array for no tools", () => { + const result = convertTools([]); + expect(result).toEqual([]); + }); + + it("preserves nested parameter schema properties", () => { + const tools: ToolContract[] = [ + { + name: "search", + description: "Search code", + parameters: { + type: "object", + properties: { + query: { type: "string", description: "Search query" }, + options: { + type: "object", + properties: { + limit: { type: "number", description: "Max results", default: 10 }, + }, + }, + }, + required: ["query"], + }, + execute: async () => ({ content: "" }), + }, + ]; + + const result = convertTools(tools); + expect(result[0]?.function.parameters.properties?.options).toEqual({ + type: "object", + properties: { + limit: { type: "number", description: "Max results", default: 10 }, + }, + }); + }); +}); diff --git a/packages/provider-openai-compat/src/convert-tools.ts b/packages/provider-openai-compat/src/convert-tools.ts new file mode 100644 index 0000000..65416bb --- /dev/null +++ b/packages/provider-openai-compat/src/convert-tools.ts @@ -0,0 +1,25 @@ +import type { ToolContract, ToolParameterSchema } from "@dispatch/kernel"; + +export interface OpenAITool { + readonly type: "function"; + readonly function: { + readonly name: string; + readonly description: string; + readonly parameters: ToolParameterSchema; + }; +} + +export function convertTools(tools: readonly ToolContract[]): OpenAITool[] { + return tools.map(convertTool); +} + +function convertTool(tool: ToolContract): OpenAITool { + return { + type: "function", + function: { + name: tool.name, + description: tool.description, + parameters: tool.parameters, + }, + }; +} diff --git a/packages/provider-openai-compat/src/extension.ts b/packages/provider-openai-compat/src/extension.ts new file mode 100644 index 0000000..1026381 --- /dev/null +++ b/packages/provider-openai-compat/src/extension.ts @@ -0,0 +1,41 @@ +import type { ApiKeyCredentials, Extension, HostAPI, Manifest } from "@dispatch/kernel"; +import { createOpenAICompatProvider } from "./provider.js"; + +export const manifest: Manifest = { + id: "provider-openai-compat", + name: "OpenAI-Compatible Provider", + version: "0.0.0", + apiVersion: "^0.1.0", + trust: "bundled", + activation: "eager", + capabilities: { network: true }, + contributes: { providers: ["openai-compat"] }, +}; + +export function activate(host: HostAPI): void { + const apiKey = host.config.get<string>("provider.openai-compat.apiKey"); + const baseURL = host.config.get<string>("provider.openai-compat.baseURL"); + const model = host.config.get<string>("provider.openai-compat.model") ?? "deepseek-v4-flash"; + + if (!apiKey) { + host.logger.warn( + "provider-openai-compat: no API key configured (provider.openai-compat.apiKey). Provider not registered.", + ); + return; + } + + const credentials: ApiKeyCredentials = { + type: "api-key", + apiKey, + ...(baseURL !== undefined ? { baseURL } : {}), + }; + + const provider = createOpenAICompatProvider({ credentials, model }); + host.defineProvider(provider); + host.logger.info(`provider-openai-compat: registered (model=${model})`); +} + +export const extension: Extension = { + manifest, + activate, +}; diff --git a/packages/provider-openai-compat/src/index.ts b/packages/provider-openai-compat/src/index.ts new file mode 100644 index 0000000..f35f2e9 --- /dev/null +++ b/packages/provider-openai-compat/src/index.ts @@ -0,0 +1,8 @@ +export type { OpenAIMessage, OpenAIToolCall } from "./convert-messages.js"; +export { convertMessages } from "./convert-messages.js"; +export type { OpenAITool } from "./convert-tools.js"; +export { convertTools } from "./convert-tools.js"; +export { activate, extension, manifest } from "./extension.js"; +export { parseSSELines } from "./parse-sse.js"; +export type { CreateOpenAICompatProviderOpts } from "./provider.js"; +export { createOpenAICompatProvider } from "./provider.js"; diff --git a/packages/provider-openai-compat/src/parse-sse.test.ts b/packages/provider-openai-compat/src/parse-sse.test.ts new file mode 100644 index 0000000..15d652e --- /dev/null +++ b/packages/provider-openai-compat/src/parse-sse.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, it } from "vitest"; +import { parseSSELines } from "./parse-sse.js"; + +describe("parseSSELines", () => { + it("parses text delta events", () => { + const lines = [ + 'data: {"id":"chatcmpl-1","choices":[{"delta":{"content":"Hello"},"index":0}]}', + 'data: {"id":"chatcmpl-1","choices":[{"delta":{"content":" world"},"index":0}]}', + 'data: {"id":"chatcmpl-1","choices":[{"delta":{},"finish_reason":"stop","index":0}]}', + "data: [DONE]", + ]; + + const events = parseSSELines(lines); + expect(events).toEqual([ + { type: "text-delta", delta: "Hello" }, + { type: "text-delta", delta: " world" }, + { type: "finish", reason: "stop" }, + ]); + }); + + it("parses a fragmented tool_call across chunks", () => { + const lines = [ + 'data: {"id":"chatcmpl-2","choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_abc","function":{"name":"read_file","arguments":""}}]},"index":0}]}', + 'data: {"id":"chatcmpl-2","choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\\"path\\""}}]},"index":0}]}', + 'data: {"id":"chatcmpl-2","choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":":\\"main.ts\\"}"}}]},"index":0}]}', + 'data: {"id":"chatcmpl-2","choices":[{"delta":{},"finish_reason":"tool_calls","index":0}]}', + "data: [DONE]", + ]; + + const events = parseSSELines(lines); + expect(events).toEqual([ + { + type: "tool-call", + toolCallId: "call_abc", + toolName: "read_file", + input: { path: "main.ts" }, + }, + { type: "finish", reason: "tool_calls" }, + ]); + }); + + it("parses multiple tool_calls in one response", () => { + const lines = [ + 'data: {"id":"chatcmpl-3","choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"read_file","arguments":""}}]},"index":0}]}', + 'data: {"id":"chatcmpl-3","choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\\"path\\":\\"a.ts\\"}"}}]},"index":0}]}', + 'data: {"id":"chatcmpl-3","choices":[{"delta":{"tool_calls":[{"index":1,"id":"call_2","function":{"name":"read_file","arguments":""}}]},"index":0}]}', + 'data: {"id":"chatcmpl-3","choices":[{"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\\"path\\":\\"b.ts\\"}"}}]},"index":0}]}', + 'data: {"id":"chatcmpl-3","choices":[{"delta":{},"finish_reason":"tool_calls","index":0}]}', + "data: [DONE]", + ]; + + const events = parseSSELines(lines); + expect(events).toEqual([ + { type: "tool-call", toolCallId: "call_1", toolName: "read_file", input: { path: "a.ts" } }, + { type: "tool-call", toolCallId: "call_2", toolName: "read_file", input: { path: "b.ts" } }, + { type: "finish", reason: "tool_calls" }, + ]); + }); + + it("parses usage from the final chunk", () => { + const lines = [ + 'data: {"id":"chatcmpl-4","choices":[{"delta":{"content":"Hi"},"index":0}]}', + 'data: {"id":"chatcmpl-4","choices":[{"delta":{},"finish_reason":"stop","index":0}]}', + 'data: {"id":"chatcmpl-4","usage":{"prompt_tokens":10,"completion_tokens":5}}', + "data: [DONE]", + ]; + + const events = parseSSELines(lines); + expect(events).toEqual([ + { type: "text-delta", delta: "Hi" }, + { type: "finish", reason: "stop" }, + { + type: "usage", + usage: { + inputTokens: 10, + outputTokens: 5, + cacheReadTokens: undefined, + cacheWriteTokens: undefined, + }, + }, + ]); + }); + + it("parses reasoning_content deltas", () => { + const lines = [ + 'data: {"id":"chatcmpl-5","choices":[{"delta":{"reasoning_content":"Let me think..."},"index":0}]}', + 'data: {"id":"chatcmpl-5","choices":[{"delta":{"content":"Here is my answer."},"index":0}]}', + 'data: {"id":"chatcmpl-5","choices":[{"delta":{},"finish_reason":"stop","index":0}]}', + "data: [DONE]", + ]; + + const events = parseSSELines(lines); + expect(events).toEqual([ + { type: "reasoning-delta", delta: "Let me think..." }, + { type: "text-delta", delta: "Here is my answer." }, + { type: "finish", reason: "stop" }, + ]); + }); + + it("handles invalid JSON gracefully", () => { + const lines = [ + "data: {invalid json}", + 'data: {"id":"chatcmpl-6","choices":[{"delta":{"content":"ok"},"index":0}]}', + "data: [DONE]", + ]; + + const events = parseSSELines(lines); + expect(events).toHaveLength(2); + expect(events[0]?.type).toBe("error"); + expect(events[1]).toEqual({ type: "text-delta", delta: "ok" }); + }); + + it("ignores non-data lines", () => { + const lines = [ + "event: message", + ": comment line", + 'data: {"id":"chatcmpl-7","choices":[{"delta":{"content":"hi"},"index":0}]}', + "", + "data: [DONE]", + ]; + + const events = parseSSELines(lines); + expect(events).toEqual([{ type: "text-delta", delta: "hi" }]); + }); + + it("stops at [DONE] sentinel", () => { + const lines = [ + 'data: {"id":"chatcmpl-8","choices":[{"delta":{"content":"before"},"index":0}]}', + "data: [DONE]", + 'data: {"id":"chatcmpl-8","choices":[{"delta":{"content":"after"},"index":0}]}', + ]; + + const events = parseSSELines(lines); + expect(events).toEqual([{ type: "text-delta", delta: "before" }]); + }); + + it("handles a complete turn with text, tool call, usage, and finish", () => { + const lines = [ + 'data: {"id":"chatcmpl-9","choices":[{"delta":{"content":"Let me check."},"index":0}]}', + 'data: {"id":"chatcmpl-9","choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_xyz","function":{"name":"search","arguments":""}}]},"index":0}]}', + 'data: {"id":"chatcmpl-9","choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\\"query\\":"}}]},"index":0}]}', + 'data: {"id":"chatcmpl-9","choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\\"dispatch\\"}"}}]},"index":0}]}', + 'data: {"id":"chatcmpl-9","choices":[{"delta":{},"finish_reason":"tool_calls","index":0}]}', + 'data: {"id":"chatcmpl-9","usage":{"prompt_tokens":50,"completion_tokens":20}}', + "data: [DONE]", + ]; + + const events = parseSSELines(lines); + expect(events).toEqual([ + { type: "text-delta", delta: "Let me check." }, + { + type: "tool-call", + toolCallId: "call_xyz", + toolName: "search", + input: { query: "dispatch" }, + }, + { type: "finish", reason: "tool_calls" }, + { + type: "usage", + usage: { + inputTokens: 50, + outputTokens: 20, + cacheReadTokens: undefined, + cacheWriteTokens: undefined, + }, + }, + ]); + }); + + it("handles tool_call with unparseable arguments as raw string", () => { + const lines = [ + 'data: {"id":"chatcmpl-10","choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_bad","function":{"name":"foo","arguments":"not-json"}}]},"index":0}]}', + 'data: {"id":"chatcmpl-10","choices":[{"delta":{},"finish_reason":"tool_calls","index":0}]}', + "data: [DONE]", + ]; + + const events = parseSSELines(lines); + expect(events).toEqual([ + { type: "tool-call", toolCallId: "call_bad", toolName: "foo", input: "not-json" }, + { type: "finish", reason: "tool_calls" }, + ]); + }); +}); diff --git a/packages/provider-openai-compat/src/parse-sse.ts b/packages/provider-openai-compat/src/parse-sse.ts new file mode 100644 index 0000000..0c0fd66 --- /dev/null +++ b/packages/provider-openai-compat/src/parse-sse.ts @@ -0,0 +1,125 @@ +import type { ProviderEvent } from "@dispatch/kernel"; + +interface ToolCallAccumulator { + id: string; + name: string; + arguments: string; +} + +interface SSEChunkDelta { + content?: string; + reasoning_content?: string; + tool_calls?: Array<{ + index: number; + id?: string; + function?: { name?: string; arguments?: string }; + }>; +} + +interface SSEChunkChoice { + delta: SSEChunkDelta; + finish_reason?: string | null; + index: number; +} + +interface SSEChunk { + id?: string; + choices?: SSEChunkChoice[]; + usage?: { + prompt_tokens?: number; + completion_tokens?: number; + cache_read_tokens?: number; + cache_write_tokens?: number; + }; +} + +export function parseSSELines(lines: readonly string[]): ProviderEvent[] { + const events: ProviderEvent[] = []; + const toolCalls = new Map<number, ToolCallAccumulator>(); + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed.startsWith("data:")) continue; + + const data = trimmed.slice(5).trim(); + if (data === "[DONE]") break; + + let chunk: SSEChunk; + try { + chunk = JSON.parse(data) as SSEChunk; + } catch { + events.push({ type: "error", message: `Invalid JSON in SSE data: ${data}` }); + continue; + } + + if (chunk.choices) { + for (const choice of chunk.choices) { + const delta = choice.delta; + + if (delta.content) { + events.push({ type: "text-delta", delta: delta.content }); + } + + if (delta.reasoning_content) { + events.push({ type: "reasoning-delta", delta: delta.reasoning_content }); + } + + if (delta.tool_calls) { + for (const tc of delta.tool_calls) { + const existing = toolCalls.get(tc.index); + if (existing) { + if (tc.function?.arguments) { + existing.arguments += tc.function.arguments; + } + } else { + toolCalls.set(tc.index, { + id: tc.id ?? "", + name: tc.function?.name ?? "", + arguments: tc.function?.arguments ?? "", + }); + } + } + } + + if (choice.finish_reason) { + const sortedIndices = [...toolCalls.keys()].sort((a, b) => a - b); + for (const idx of sortedIndices) { + const acc = toolCalls.get(idx); + if (!acc) continue; + let input: unknown; + try { + input = JSON.parse(acc.arguments); + } catch { + input = acc.arguments; + } + events.push({ + type: "tool-call", + toolCallId: acc.id, + toolName: acc.name, + input, + }); + } + events.push({ type: "finish", reason: choice.finish_reason }); + } + } + } + + if (chunk.usage) { + events.push({ + type: "usage", + usage: { + inputTokens: chunk.usage.prompt_tokens ?? 0, + outputTokens: chunk.usage.completion_tokens ?? 0, + ...(chunk.usage.cache_read_tokens !== undefined + ? { cacheReadTokens: chunk.usage.cache_read_tokens } + : {}), + ...(chunk.usage.cache_write_tokens !== undefined + ? { cacheWriteTokens: chunk.usage.cache_write_tokens } + : {}), + }, + }); + } + } + + return events; +} diff --git a/packages/provider-openai-compat/src/provider.ts b/packages/provider-openai-compat/src/provider.ts new file mode 100644 index 0000000..8f0ddda --- /dev/null +++ b/packages/provider-openai-compat/src/provider.ts @@ -0,0 +1,30 @@ +import type { + ApiKeyCredentials, + ChatMessage, + ProviderContract, + ProviderStreamOptions, + ToolContract, +} from "@dispatch/kernel"; +import { streamChat } from "./stream.js"; + +export interface CreateOpenAICompatProviderOpts { + readonly credentials: ApiKeyCredentials; + readonly model: string; +} + +export function createOpenAICompatProvider(opts: CreateOpenAICompatProviderOpts): ProviderContract { + const config = { + baseURL: opts.credentials.baseURL ?? "https://opencode.ai/zen/go/v1", + apiKey: opts.credentials.apiKey, + model: opts.model, + }; + + return { + id: "openai-compat", + stream: ( + messages: readonly ChatMessage[], + tools: readonly ToolContract[], + streamOpts?: ProviderStreamOptions, + ) => streamChat(config, messages, tools, streamOpts), + }; +} diff --git a/packages/provider-openai-compat/src/stream.ts b/packages/provider-openai-compat/src/stream.ts new file mode 100644 index 0000000..7021120 --- /dev/null +++ b/packages/provider-openai-compat/src/stream.ts @@ -0,0 +1,202 @@ +import type { + ChatMessage, + ProviderEvent, + ProviderStreamOptions, + ToolContract, +} from "@dispatch/kernel"; +import { convertMessages, type OpenAIMessage } from "./convert-messages.js"; +import { convertTools, type OpenAITool } from "./convert-tools.js"; + +export interface StreamConfig { + readonly baseURL: string; + readonly apiKey: string; + readonly model: string; +} + +export async function* streamChat( + config: StreamConfig, + messages: readonly ChatMessage[], + tools: readonly ToolContract[], + opts?: ProviderStreamOptions, +): AsyncIterable<ProviderEvent> { + const openaiMessages = convertMessages(messages); + const openaiTools = convertTools(tools); + + const systemPrompt = opts?.systemPrompt; + const finalMessages: OpenAIMessage[] = systemPrompt + ? [{ role: "system", content: systemPrompt }, ...openaiMessages] + : openaiMessages; + + const body: Record<string, unknown> = { + model: opts?.model ?? config.model, + messages: finalMessages, + stream: true, + }; + + if (openaiTools.length > 0) { + body.tools = openaiTools satisfies OpenAITool[]; + } + if (opts?.temperature !== undefined) { + body.temperature = opts.temperature; + } + if (opts?.maxTokens !== undefined) { + body.max_tokens = opts.maxTokens; + } + + let response: Response; + try { + response = await fetch(`${config.baseURL}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${config.apiKey}`, + }, + body: JSON.stringify(body), + }); + } catch (err) { + yield { + type: "error", + message: err instanceof Error ? err.message : String(err), + retryable: true, + }; + return; + } + + if (!response.ok) { + const text = await response.text().catch(() => "unknown"); + yield { + type: "error", + message: `HTTP ${response.status}: ${text}`, + code: String(response.status), + retryable: response.status >= 500 || response.status === 429, + }; + return; + } + + if (!response.body) { + yield { type: "error", message: "Response body is null" }; + return; + } + + yield* readSSEStream(response.body); +} + +async function* readSSEStream(body: ReadableStream<Uint8Array>): AsyncIterable<ProviderEvent> { + const reader = body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + const toolCalls = new Map<number, { id: string; name: string; arguments: string }>(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed.startsWith("data:")) continue; + + const data = trimmed.slice(5).trim(); + if (data === "[DONE]") return; + + let chunk: Record<string, unknown>; + try { + chunk = JSON.parse(data); + } catch { + yield { type: "error", message: `Invalid JSON in SSE data: ${data}` }; + continue; + } + + const choices = chunk.choices as + | Array<{ + delta: Record<string, unknown>; + finish_reason?: string | null; + }> + | undefined; + + if (choices) { + for (const choice of choices) { + const delta = choice.delta; + + if (typeof delta.content === "string" && delta.content) { + yield { type: "text-delta", delta: delta.content }; + } + + if (typeof delta.reasoning_content === "string" && delta.reasoning_content) { + yield { type: "reasoning-delta", delta: delta.reasoning_content }; + } + + const tcs = delta.tool_calls as + | Array<{ + index: number; + id?: string; + function?: { name?: string; arguments?: string }; + }> + | undefined; + + if (tcs) { + for (const tc of tcs) { + const existing = toolCalls.get(tc.index); + if (existing) { + if (tc.function?.arguments) { + existing.arguments += tc.function.arguments; + } + } else { + toolCalls.set(tc.index, { + id: tc.id ?? "", + name: tc.function?.name ?? "", + arguments: tc.function?.arguments ?? "", + }); + } + } + } + + if (choice.finish_reason) { + const sortedIndices = [...toolCalls.keys()].sort((a, b) => a - b); + for (const idx of sortedIndices) { + const acc = toolCalls.get(idx); + if (!acc) continue; + let input: unknown; + try { + input = JSON.parse(acc.arguments); + } catch { + input = acc.arguments; + } + yield { + type: "tool-call", + toolCallId: acc.id, + toolName: acc.name, + input, + }; + } + yield { type: "finish", reason: choice.finish_reason }; + } + } + } + + const usage = chunk.usage as + | { + prompt_tokens?: number; + completion_tokens?: number; + } + | undefined; + + if (usage) { + yield { + type: "usage", + usage: { + inputTokens: usage.prompt_tokens ?? 0, + outputTokens: usage.completion_tokens ?? 0, + }, + }; + } + } + } + } finally { + reader.releaseLock(); + } +} diff --git a/packages/provider-openai-compat/tsconfig.json b/packages/provider-openai-compat/tsconfig.json new file mode 100644 index 0000000..ff99a43 --- /dev/null +++ b/packages/provider-openai-compat/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "rootDir": "src", "outDir": "dist", "composite": true }, + "include": ["src/**/*.ts"], + "references": [{ "path": "../kernel" }] +} diff --git a/packages/storage-sqlite/package.json b/packages/storage-sqlite/package.json new file mode 100644 index 0000000..1f60e0a --- /dev/null +++ b/packages/storage-sqlite/package.json @@ -0,0 +1,11 @@ +{ + "name": "@dispatch/storage-sqlite", + "version": "0.0.0", + "type": "module", + "private": true, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "dependencies": { + "@dispatch/kernel": "workspace:*" + } +} diff --git a/packages/storage-sqlite/src/extension.ts b/packages/storage-sqlite/src/extension.ts new file mode 100644 index 0000000..63a71af --- /dev/null +++ b/packages/storage-sqlite/src/extension.ts @@ -0,0 +1,17 @@ +import type { Extension, HostAPI, Manifest } from "@dispatch/kernel"; + +export const manifest: Manifest = { + id: "storage-sqlite", + name: "SQLite Storage Backend", + version: "0.0.0", + apiVersion: "^0.1.0", + trust: "bundled", + capabilities: { db: true }, + contributes: { services: ["storage"] }, + activation: "eager", +}; + +export const extension: Extension = { + manifest, + activate: async (_host: HostAPI) => {}, +}; diff --git a/packages/storage-sqlite/src/index.ts b/packages/storage-sqlite/src/index.ts new file mode 100644 index 0000000..4fbedfc --- /dev/null +++ b/packages/storage-sqlite/src/index.ts @@ -0,0 +1,5 @@ +export { extension, manifest } from "./extension.js"; +export type { Migration } from "./migrate.js"; +export { computePending } from "./migrate.js"; +export type { SqliteStorageBackend, StorageFactory } from "./storage.js"; +export { createSqliteStorage } from "./storage.js"; diff --git a/packages/storage-sqlite/src/migrate.ts b/packages/storage-sqlite/src/migrate.ts new file mode 100644 index 0000000..bdd32b8 --- /dev/null +++ b/packages/storage-sqlite/src/migrate.ts @@ -0,0 +1,14 @@ +export interface Migration { + readonly version: number; + readonly name: string; + readonly up: string; +} + +export function computePending( + appliedVersions: ReadonlySet<number>, + migrations: readonly Migration[], +): readonly Migration[] { + return migrations + .filter((m) => !appliedVersions.has(m.version)) + .sort((a, b) => a.version - b.version); +} diff --git a/packages/storage-sqlite/src/storage.test.ts b/packages/storage-sqlite/src/storage.test.ts new file mode 100644 index 0000000..fc14498 --- /dev/null +++ b/packages/storage-sqlite/src/storage.test.ts @@ -0,0 +1,218 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { Migration } from "./migrate.js"; +import { computePending } from "./migrate.js"; +import type { SqliteStorageBackend } from "./storage.js"; +import { createSqliteStorage } from "./storage.js"; + +describe("computePending (pure)", () => { + it("returns all migrations when none applied", () => { + const migrations: Migration[] = [ + { version: 2, name: "b", up: "" }, + { version: 1, name: "a", up: "" }, + ]; + const result = computePending(new Set(), migrations); + expect(result.map((m) => m.version)).toEqual([1, 2]); + }); + + it("skips already-applied versions", () => { + const migrations: Migration[] = [ + { version: 1, name: "a", up: "" }, + { version: 2, name: "b", up: "" }, + { version: 3, name: "c", up: "" }, + ]; + const result = computePending(new Set([1, 3]), migrations); + expect(result.map((m) => m.version)).toEqual([2]); + }); + + it("returns empty when all applied", () => { + const migrations: Migration[] = [{ version: 1, name: "a", up: "" }]; + const result = computePending(new Set([1]), migrations); + expect(result).toEqual([]); + }); + + it("sorts by version ascending", () => { + const migrations: Migration[] = [ + { version: 3, name: "c", up: "" }, + { version: 1, name: "a", up: "" }, + { version: 2, name: "b", up: "" }, + ]; + const result = computePending(new Set(), migrations); + expect(result.map((m) => m.version)).toEqual([1, 2, 3]); + }); +}); + +describe("createSqliteStorage", () => { + let backend: SqliteStorageBackend; + + beforeEach(() => { + backend = createSqliteStorage({ path: ":memory:" }); + }); + + afterEach(() => { + backend.close(); + }); + + describe("StorageNamespace get/set/delete/has", () => { + it("returns null for missing key", async () => { + const ns = backend.storage("test"); + expect(await ns.get("missing")).toBeNull(); + }); + + it("roundtrips set then get", async () => { + const ns = backend.storage("test"); + await ns.set("key1", "value1"); + expect(await ns.get("key1")).toBe("value1"); + }); + + it("overwrites existing value", async () => { + const ns = backend.storage("test"); + await ns.set("key1", "v1"); + await ns.set("key1", "v2"); + expect(await ns.get("key1")).toBe("v2"); + }); + + it("has returns false for missing, true for existing", async () => { + const ns = backend.storage("test"); + expect(await ns.has("key1")).toBe(false); + await ns.set("key1", "val"); + expect(await ns.has("key1")).toBe(true); + }); + + it("delete removes the key", async () => { + const ns = backend.storage("test"); + await ns.set("key1", "val"); + await ns.delete("key1"); + expect(await ns.get("key1")).toBeNull(); + expect(await ns.has("key1")).toBe(false); + }); + + it("delete on missing key is a no-op", async () => { + const ns = backend.storage("test"); + await ns.delete("nonexistent"); + }); + }); + + describe("keys", () => { + it("returns all keys in namespace", async () => { + const ns = backend.storage("test"); + await ns.set("a", "1"); + await ns.set("b", "2"); + await ns.set("c", "3"); + const keys = await ns.keys(); + expect(keys.toSorted()).toEqual(["a", "b", "c"]); + }); + + it("filters by prefix", async () => { + const ns = backend.storage("test"); + await ns.set("foo:1", "a"); + await ns.set("foo:2", "b"); + await ns.set("bar:1", "c"); + const keys = await ns.keys("foo"); + expect(keys.toSorted()).toEqual(["foo:1", "foo:2"]); + }); + + it("returns empty for no matches", async () => { + const ns = backend.storage("test"); + await ns.set("a", "1"); + expect(await ns.keys("zzz")).toEqual([]); + }); + }); + + describe("namespace isolation", () => { + it("same key in different namespaces does not collide", async () => { + const ns1 = backend.storage("ns1"); + const ns2 = backend.storage("ns2"); + await ns1.set("key", "value1"); + await ns2.set("key", "value2"); + expect(await ns1.get("key")).toBe("value1"); + expect(await ns2.get("key")).toBe("value2"); + }); + + it("delete in one namespace does not affect another", async () => { + const ns1 = backend.storage("ns1"); + const ns2 = backend.storage("ns2"); + await ns1.set("key", "v1"); + await ns2.set("key", "v2"); + await ns1.delete("key"); + expect(await ns1.has("key")).toBe(false); + expect(await ns2.get("key")).toBe("v2"); + }); + + it("keys are scoped to namespace", async () => { + const ns1 = backend.storage("ns1"); + const ns2 = backend.storage("ns2"); + await ns1.set("a", "1"); + await ns1.set("b", "2"); + await ns2.set("c", "3"); + expect((await ns1.keys()).toSorted()).toEqual(["a", "b"]); + expect(await ns2.keys()).toEqual(["c"]); + }); + }); + + describe("migrate", () => { + it("runs pending migrations in order", async () => { + const migrations: Migration[] = [ + { version: 1, name: "create_foo", up: "CREATE TABLE foo (id INTEGER PRIMARY KEY);" }, + { version: 2, name: "create_bar", up: "CREATE TABLE bar (id INTEGER PRIMARY KEY);" }, + ]; + await backend.migrate("ext1", migrations); + + const ns = backend.storage("ext1"); + await ns.set("test", "val"); + expect(await ns.get("test")).toBe("val"); + }); + + it("skips already-applied migrations", async () => { + const migrations: Migration[] = [ + { version: 1, name: "create_foo", up: "CREATE TABLE foo (id INTEGER PRIMARY KEY);" }, + ]; + await backend.migrate("ext1", migrations); + await backend.migrate("ext1", [ + ...migrations, + { version: 2, name: "create_bar", up: "CREATE TABLE bar (id INTEGER PRIMARY KEY);" }, + ]); + }); + + it("is idempotent — calling migrate twice with same migrations is safe", async () => { + const migrations: Migration[] = [ + { version: 1, name: "create_foo", up: "CREATE TABLE foo (id INTEGER PRIMARY KEY);" }, + ]; + await backend.migrate("ext1", migrations); + await backend.migrate("ext1", migrations); + }); + + it("migrations are per-namespace", async () => { + const m1: Migration[] = [ + { version: 1, name: "create_t1", up: "CREATE TABLE t1 (id INTEGER PRIMARY KEY);" }, + ]; + const m2: Migration[] = [ + { version: 1, name: "create_t2", up: "CREATE TABLE t2 (id INTEGER PRIMARY KEY);" }, + ]; + await backend.migrate("ext1", m1); + await backend.migrate("ext2", m2); + }); + }); +}); + +describe("createSqliteStorage — persistence across reopen", () => { + it("migrations survive close and reopen", async () => { + const tmpPath = `/tmp/storage-sqlite-test-${Date.now()}.db`; + const migrations: Migration[] = [ + { version: 1, name: "create_foo", up: "CREATE TABLE foo (id INTEGER PRIMARY KEY);" }, + ]; + + const first = createSqliteStorage({ path: tmpPath }); + await first.migrate("ext1", migrations); + first.close(); + + const second = createSqliteStorage({ path: tmpPath }); + await second.migrate("ext1", [ + ...migrations, + { version: 2, name: "create_bar", up: "CREATE TABLE bar (id INTEGER PRIMARY KEY);" }, + ]); + second.close(); + + const { unlinkSync } = await import("node:fs"); + unlinkSync(tmpPath); + }); +}); diff --git a/packages/storage-sqlite/src/storage.ts b/packages/storage-sqlite/src/storage.ts new file mode 100644 index 0000000..2499664 --- /dev/null +++ b/packages/storage-sqlite/src/storage.ts @@ -0,0 +1,92 @@ +import { Database } from "bun:sqlite"; +import type { StorageNamespace } from "@dispatch/kernel"; +import { computePending, type Migration } from "./migrate.js"; + +export interface SqliteStorageBackend { + readonly storage: (namespace: string) => StorageNamespace; + readonly migrate: (namespace: string, migrations: readonly Migration[]) => Promise<void>; + readonly close: () => void; +} + +export type StorageFactory = (opts: { path: string }) => SqliteStorageBackend; + +export function createSqliteStorage(opts: { path: string }): SqliteStorageBackend { + const db = new Database(opts.path); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec(` + CREATE TABLE IF NOT EXISTS kv ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (namespace, key) + ); + `); + db.exec(` + CREATE TABLE IF NOT EXISTS _migrations ( + namespace TEXT NOT NULL, + version INTEGER NOT NULL, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (namespace, version) + ); + `); + + const getStmt = db.prepare("SELECT value FROM kv WHERE namespace = ?1 AND key = ?2"); + const setStmt = db.prepare( + "INSERT OR REPLACE INTO kv (namespace, key, value) VALUES (?1, ?2, ?3)", + ); + const deleteStmt = db.prepare("DELETE FROM kv WHERE namespace = ?1 AND key = ?2"); + const hasStmt = db.prepare("SELECT 1 FROM kv WHERE namespace = ?1 AND key = ?2"); + const keysAllStmt = db.prepare("SELECT key FROM kv WHERE namespace = ?1"); + const keysPrefixStmt = db.prepare("SELECT key FROM kv WHERE namespace = ?1 AND key LIKE ?2"); + const appliedVersionsStmt = db.prepare("SELECT version FROM _migrations WHERE namespace = ?1"); + const recordMigrationStmt = db.prepare( + "INSERT INTO _migrations (namespace, version, name) VALUES (?1, ?2, ?3)", + ); + + function storage(namespace: string): StorageNamespace { + return { + get: async (key: string) => { + const row = getStmt.get(namespace, key) as { value: string } | null; + return row?.value ?? null; + }, + set: async (key: string, value: string) => { + setStmt.run(namespace, key, value); + }, + delete: async (key: string) => { + deleteStmt.run(namespace, key); + }, + has: async (key: string) => { + return hasStmt.get(namespace, key) !== null; + }, + keys: async (prefix?: string) => { + if (prefix !== undefined) { + const rows = keysPrefixStmt.all(namespace, `${prefix}%`) as { key: string }[]; + return rows.map((r) => r.key); + } + const rows = keysAllStmt.all(namespace) as { key: string }[]; + return rows.map((r) => r.key); + }, + }; + } + + async function migrate(namespace: string, migrations: readonly Migration[]): Promise<void> { + const rows = appliedVersionsStmt.all(namespace) as { version: number }[]; + const applied = new Set(rows.map((r) => r.version)); + const pending = computePending(applied, migrations); + + for (const m of pending) { + const runInTransaction = db.transaction(() => { + db.exec(m.up); + recordMigrationStmt.run(namespace, m.version, m.name); + }); + runInTransaction(); + } + } + + function close(): void { + db.close(); + } + + return { storage, migrate, close }; +} diff --git a/packages/storage-sqlite/tsconfig.json b/packages/storage-sqlite/tsconfig.json new file mode 100644 index 0000000..ff99a43 --- /dev/null +++ b/packages/storage-sqlite/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "rootDir": "src", "outDir": "dist", "composite": true }, + "include": ["src/**/*.ts"], + "references": [{ "path": "../kernel" }] +} @@ -10,31 +10,35 @@ drive fan-out); extension **loading is dynamic** (manifests via the host). ## Legend: [ ] todo [~] in progress [x] done (verified + committed) ### Kernel -- [x] **contracts** — the ABI (conversation, tool, provider, auth, dispatch, hooks, extension/HostAPI, runtime, events). `fd855ff` -- [x] **bus** — event/hook/service bus (pure dispatch + stateful shell). `669c269` -- [ ] **runtime** — `runTurn` turn loop (provider+tools→events; dispatch policy §3.3). prompt ready: `prompts/kernel-runtime.md` -- [ ] **host** — extension discovery → DAG resolve → activate → registries; builds `HostAPI` (wraps bus). Registries: tools, providers, auth, services, storage, migrations. +- [x] **contracts** — the ABI. `fd855ff` (+ `974ce6f` RunTurnInput tabId/turnId/providerOpts, FinishReason) +- [x] **bus** — event/hook/service bus. `669c269` +- [x] **runtime** — `runTurn` turn loop (dispatch policy §3.3), 16 tests. `ae22da5` +- [~] **host** — discovery→DAG→activate→registries; builds HostAPI (wraps bus). RUNNING. ### Core extensions (each ships a real manifest, loaded via the host) -- [ ] **storage-sqlite** — concrete backend behind `host.storage` (bun:sqlite). -- [ ] **conversation-store** — append-only turn/chunk persistence on host.storage (multi-turn = target B). -- [ ] **auth-apikey** — resolves `{ apiKey, baseURL }` from `.env`. -- [ ] **provider-openai-compat** — OpenAI-compatible streaming → ProviderEvents (OpenCode Go). -- [ ] **session-orchestrator** — on message: load history → resolve provider/tools → `runTurn` → persist. -- [ ] **transport-http** — Hono `/chat` route; composes the above via host/bus. +- [~] **storage-sqlite** — concrete backend behind host.storage (bun:sqlite). RUNNING. +- [~] **auth-apikey** — resolves `{ apiKey, baseURL }` from env (.env). RUNNING. +- [~] **provider-openai-compat** — OpenAI-compatible streaming → ProviderEvents (OpenCode Go flash). RUNNING. +- [ ] **conversation-store** — append-only turn/chunk persistence on host.storage (multi-turn = target B). (after storage-sqlite) +- [ ] **session-orchestrator** — on message: load history → resolve provider/tools → runTurn → persist. +- [ ] **transport-http** — Hono `/chat` route; composes via host/bus. ### Integration - [ ] **host-bin** — boot: load config, discover+activate extensions, start transport. -- [ ] **curl smoke test** — `curl -d '{...}' /chat` → real response from flash; verify multi-turn (turn 2 sees turn 1). +- [ ] **curl smoke test** — `curl -d '{...}' /chat` → real response from flash; verify multi-turn. -## Notes / open contract-watch -- Provider needs resolved credentials/model: `ProviderContract.stream` currently - has no creds param. Watch when building provider/auth — may be a contract change - (provider closes over creds at construction, OR add to stream opts). -- `HostAPI.addFilter` lacks optional `{ priority }` (bus supports it). Decide if - HostAPI should expose it (CR from bus report). +## Parallel batch in flight +host + storage-sqlite + auth-apikey + provider-openai-compat (disjoint files). +Shared resource: root `tsconfig.json` references — agents NOTE only; orchestrator +adds the 3 package refs after they land (avoids write race). -## Build order (dependency-topological) +## Open contract-watch +- Provider credentials/model: how provider receives creds (construction vs stream + opts). auth-apikey + provider agents building in parallel — RECONCILE on return. +- `host` deps injection shape (storageFactory, logger, config, secrets, perms) — + defines what host-bin must wire. + +## Build order contracts → bus → runtime → host → storage-sqlite → conversation-store → auth-apikey → provider-openai-compat → session-orchestrator → transport-http → host-bin → curl. diff --git a/tsconfig.json b/tsconfig.json index e504b9b..fa41981 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,4 +1,17 @@ { "files": [], - "references": [{ "path": "./packages/kernel" }] + "references": [ + { + "path": "./packages/auth-apikey" + }, + { + "path": "./packages/kernel" + }, + { + "path": "./packages/provider-openai-compat" + }, + { + "path": "./packages/storage-sqlite" + } + ] } diff --git a/vitest.config.ts b/vitest.config.ts index e74aab6..f682bbf 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,6 +2,9 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { - projects: ["packages/*"], + // Packages whose code imports Bun-only modules (e.g. `bun:sqlite`) can't run + // under Vite's Node transform — they test via `bun test` (see `test:bun`). + // Everything else runs here under vitest. + projects: ["packages/*", "!packages/storage-sqlite"], }, }); |
