From 55a6fcdd3f5b3c55712e5cfc9dd4d994da38d4c8 Mon Sep 17 00:00:00 2001 From: Dax Raad Date: Wed, 28 May 2025 12:53:22 -0400 Subject: add provider_list --- js/src/app/config.ts | 35 ++++++++----- js/src/bun/index.ts | 25 ++++++++++ js/src/global/index.ts | 20 ++++++++ js/src/index.ts | 17 +++++-- js/src/llm/llm.ts | 109 +++++++++++++++++++++++++++++++---------- js/src/llm/models/anthropic.ts | 0 js/src/llm/models/index.ts | 1 + js/src/llm/models/model.ts | 11 +++++ js/src/server/server.ts | 35 ++++++++++++- js/src/session/session.ts | 58 +++++++++++----------- 10 files changed, 238 insertions(+), 73 deletions(-) create mode 100644 js/src/bun/index.ts create mode 100644 js/src/global/index.ts create mode 100644 js/src/llm/models/anthropic.ts create mode 100644 js/src/llm/models/index.ts create mode 100644 js/src/llm/models/model.ts (limited to 'js/src') diff --git a/js/src/app/config.ts b/js/src/app/config.ts index 947298c67..77a1f606f 100644 --- a/js/src/app/config.ts +++ b/js/src/app/config.ts @@ -1,25 +1,34 @@ import path from "node:path"; import { Log } from "../util/log"; import { z } from "zod"; +import { LLM } from "../llm/llm"; export namespace Config { const log = Log.create({ service: "config" }); + export const Model = z.object({ + name: z.string().optional(), + cost: z.object({ + input: z.number(), + inputCached: z.number(), + output: z.number(), + outputCached: z.number(), + }), + contextWindow: z.number(), + maxTokens: z.number(), + attachment: z.boolean(), + }); + export type Model = z.output; + + export const Provider = z.object({ + options: z.record(z.string(), z.any()).optional(), + models: z.record(z.string(), Model).optional(), + }); + export type Provider = z.output; + export const Info = z .object({ - providers: z - .object({ - anthropic: z - .object({ - apiKey: z.string().optional(), - headers: z.record(z.string(), z.string()).optional(), - baseURL: z.string().optional(), - }) - .strict() - .optional(), - }) - .strict() - .optional(), + providers: z.record(z.string(), Provider).optional(), }) .strict(); diff --git a/js/src/bun/index.ts b/js/src/bun/index.ts new file mode 100644 index 000000000..e921c825a --- /dev/null +++ b/js/src/bun/index.ts @@ -0,0 +1,25 @@ +import path from "node:path"; +import { Log } from "../util/log"; +export namespace BunProc { + const log = Log.create({ service: "bun" }); + + export function run( + cmd: string[], + options?: Bun.SpawnOptions.OptionsObject, + ) { + const root = path.resolve(process.cwd(), process.argv0); + log.info("running", { + cmd: [root, ...cmd], + options, + }); + const result = Bun.spawnSync([root, ...cmd], { + ...options, + argv0: "bun", + env: { + ...process.env, + ...options?.env, + }, + }); + return result; + } +} diff --git a/js/src/global/index.ts b/js/src/global/index.ts new file mode 100644 index 000000000..1e097f38c --- /dev/null +++ b/js/src/global/index.ts @@ -0,0 +1,20 @@ +import envpaths from "env-paths"; +import fs from "fs/promises"; +const paths = envpaths("opencode", { + suffix: "", +}); + +await Promise.all([ + fs.mkdir(paths.config, { recursive: true }), + fs.mkdir(paths.cache, { recursive: true }), +]); + +export namespace Global { + export function config() { + return paths.config; + } + + export function cache() { + return paths.cache; + } +} diff --git a/js/src/index.ts b/js/src/index.ts index dc80206f7..3758ad718 100644 --- a/js/src/index.ts +++ b/js/src/index.ts @@ -7,6 +7,7 @@ import { Session } from "./session/session"; import cac from "cac"; import { Share } from "./share/share"; import { Storage } from "./storage/storage"; +import { LLM } from "./llm/llm"; const cli = cac("opencode"); @@ -90,9 +91,19 @@ cli } }); - const result = await Session.chat(session.id, { - type: "text", - text: message.join(" "), + const providers = await LLM.providers(); + const providerID = Object.keys(providers)[0]; + const modelID = Object.keys(providers[providerID].info.models!)[0]; + const result = await Session.chat({ + sessionID: session.id, + providerID, + modelID, + parts: [ + { + type: "text", + text: message.join(" "), + }, + ], }); for (const part of result.parts) { diff --git a/js/src/llm/llm.ts b/js/src/llm/llm.ts index 5d962219c..c0ab38530 100644 --- a/js/src/llm/llm.ts +++ b/js/src/llm/llm.ts @@ -1,9 +1,13 @@ import { App } from "../app"; import { Log } from "../util/log"; +import { mergeDeep } from "remeda"; +import path from "node:path"; -import { createAnthropic } from "@ai-sdk/anthropic"; import type { LanguageModel, Provider } from "ai"; import { NoSuchModelError } from "ai"; +import type { Config } from "../app/config"; +import { BunProc } from "../bun"; +import { Global } from "../global"; export namespace LLM { const log = Log.create({ service: "llm" }); @@ -14,17 +18,67 @@ export namespace LLM { } } + const NATIVE_PROVIDERS: Record = { + anthropic: { + models: { + "claude-sonnet-4-20250514": { + name: "Claude 4 Sonnet", + cost: { + input: 3.0, + inputCached: 3.75, + output: 15.0, + outputCached: 0.3, + }, + contextWindow: 200000, + maxTokens: 50000, + attachment: true, + }, + }, + }, + }; + + const AUTODETECT: Record = { + anthropic: ["ANTHROPIC_API_KEY"], + }; + const state = App.state("llm", async (app) => { - const providers: Provider[] = []; - - if (process.env["ANTHROPIC_API_KEY"] || app.config.providers?.anthropic) { - log.info("loaded anthropic"); - const provider = createAnthropic({ - apiKey: app.config.providers?.anthropic?.apiKey, - baseURL: app.config.providers?.anthropic?.baseURL, - headers: app.config.providers?.anthropic?.headers, - }); - providers.push(provider); + const providers: Record< + string, + { + info: Config.Provider; + instance: Provider; + } + > = {}; + + const list = mergeDeep(NATIVE_PROVIDERS, app.config.providers ?? {}); + + for (const [providerID, providerInfo] of Object.entries(list)) { + if ( + !app.config.providers?.[providerID] && + !AUTODETECT[providerID]?.some((env) => process.env[env]) + ) + continue; + const dir = path.join( + Global.cache(), + `node_modules`, + `@ai-sdk`, + providerID, + ); + if (!(await Bun.file(path.join(dir, "package.json")).exists())) { + BunProc.run(["add", "--exact", `@ai-sdk/${providerID}@alpha`], { + cwd: Global.cache(), + }); + } + const mod = await import( + path.join(Global.cache(), `node_modules`, `@ai-sdk`, providerID) + ); + const fn = mod[Object.keys(mod).find((key) => key.startsWith("create"))!]; + const loaded = fn(providerInfo.options); + log.info("loaded", { provider: providerID }); + providers[providerID] = { + info: providerInfo, + instance: loaded, + }; } return { @@ -37,23 +91,24 @@ export namespace LLM { return state().then((state) => state.providers); } - export async function findModel(model: string) { + export async function findModel(providerID: string, modelID: string) { + const key = `${providerID}/${modelID}`; const s = await state(); - if (s.models.has(model)) { - return s.models.get(model)!; - } - log.info("loading", { model }); - for (const provider of s.providers) { - try { - const match = provider.languageModel(model); - log.info("found", { model }); - s.models.set(model, match); - return match; - } catch (e) { - if (e instanceof NoSuchModelError) continue; - throw e; - } + if (s.models.has(key)) return s.models.get(key)!; + const provider = s.providers[providerID]; + if (!provider) throw new ModelNotFoundError(modelID); + log.info("loading", { + providerID, + modelID, + }); + try { + const match = provider.instance.languageModel(modelID); + log.info("found", { providerID, modelID }); + s.models.set(key, match); + return match; + } catch (e) { + if (e instanceof NoSuchModelError) throw new ModelNotFoundError(modelID); + throw e; } - throw new ModelNotFoundError(model); } } diff --git a/js/src/llm/models/anthropic.ts b/js/src/llm/models/anthropic.ts new file mode 100644 index 000000000..e69de29bb diff --git a/js/src/llm/models/index.ts b/js/src/llm/models/index.ts new file mode 100644 index 000000000..f974b4d3e --- /dev/null +++ b/js/src/llm/models/index.ts @@ -0,0 +1 @@ +export * as anthropic from "./anthropic"; diff --git a/js/src/llm/models/model.ts b/js/src/llm/models/model.ts new file mode 100644 index 000000000..e78dbb87f --- /dev/null +++ b/js/src/llm/models/model.ts @@ -0,0 +1,11 @@ +export interface ModelInfo { + cost: { + input: number; + inputCached: number; + output: number; + outputCached: number; + }; + contextWindow: number; + maxTokens: number; + attachment: boolean; +} diff --git a/js/src/server/server.ts b/js/src/server/server.ts index 410463240..57f52a37c 100644 --- a/js/src/server/server.ts +++ b/js/src/server/server.ts @@ -7,11 +7,18 @@ import { Session } from "../session/session"; import { resolver, validator as zValidator } from "hono-openapi/zod"; import { z } from "zod"; import "zod-openapi/extend"; +import { Config } from "../app/config"; +import { LLM } from "../llm/llm"; const SessionInfo = Session.Info.openapi({ ref: "Session.Info", }); +const ProviderInfo = Config.Provider.openapi({ + ref: "Provider.Info", +}); +type ProviderInfo = z.output; + export namespace Server { const log = Log.create({ service: "server" }); const PORT = 16713; @@ -156,14 +163,40 @@ export namespace Server { "json", z.object({ sessionID: z.string(), + providerID: z.string(), + modelID: z.string(), parts: z.custom(), }), ), async (c) => { const body = c.req.valid("json"); - const msg = await Session.chat(body.sessionID, ...body.parts); + const msg = await Session.chat(body); return c.json(msg); }, + ) + .post( + "/provider_list", + describeRoute({ + description: "List all providers", + responses: { + 200: { + description: "List of providers", + content: { + "application/json": { + schema: resolver(z.record(z.string(), ProviderInfo)), + }, + }, + }, + }, + }), + async (c) => { + const providers = await LLM.providers(); + const result: Record = {}; + for (const [providerID, provider] of Object.entries(providers)) { + result[providerID] = provider.info; + } + return c.json(result); + }, ); return result; diff --git a/js/src/session/session.ts b/js/src/session/session.ts index 7447f4916..06c51625f 100644 --- a/js/src/session/session.ts +++ b/js/src/session/session.ts @@ -127,19 +127,19 @@ export namespace Session { } } - export async function chat( - sessionID: string, - ...parts: UIMessagePart[] - ) { - const model = await LLM.findModel("claude-sonnet-4-20250514"); - const session = await get(sessionID); - const l = log.clone().tag("session", sessionID); + export async function chat(input: { + sessionID: string; + providerID: string; + modelID: string; + parts: UIMessagePart[]; + }) { + const l = log.clone().tag("session", input.sessionID); l.info("chatting"); - - const msgs = await messages(sessionID); + const model = await LLM.findModel(input.providerID, input.modelID); + const msgs = await messages(input.sessionID); async function write(msg: Message) { return Storage.writeJSON( - "session/message/" + sessionID + "/" + msg.id, + "session/message/" + input.sessionID + "/" + msg.id, msg, ); } @@ -155,7 +155,7 @@ export namespace Session { }, ], metadata: { - sessionID, + sessionID: input.sessionID, time: { created: Date.now(), }, @@ -171,7 +171,7 @@ export namespace Session { }); } msgs.push(system); - state().messages.set(sessionID, msgs); + state().messages.set(input.sessionID, msgs); generateText({ messages: convertToModelMessages([ { @@ -185,12 +185,12 @@ export namespace Session { }, { role: "user", - parts, + parts: input.parts, }, ]), model, }).then((result) => { - return Session.update(sessionID, (draft) => { + return Session.update(input.sessionID, (draft) => { draft.title = result.text; }); }); @@ -199,21 +199,33 @@ export namespace Session { const msg: Message = { role: "user", id: Identifier.ascending("message"), - parts, + parts: input.parts, metadata: { time: { created: Date.now(), }, - sessionID, + sessionID: input.sessionID, tool: {}, }, }; msgs.push(msg); await write(msg); + const next: Message = { + id: Identifier.ascending("message"), + role: "assistant", + parts: [], + metadata: { + time: { + created: Date.now(), + }, + sessionID: input.sessionID, + tool: {}, + }, + }; const result = streamText({ onStepFinish: (step) => { - update(sessionID, (draft) => { + update(input.sessionID, (draft) => { draft.tokens.input += step.usage.inputTokens || 0; draft.tokens.output += step.usage.outputTokens || 0; draft.tokens.reasoning += step.usage.reasoningTokens || 0; @@ -225,18 +237,6 @@ export namespace Session { tools, model, }); - const next: Message = { - id: Identifier.ascending("message"), - role: "assistant", - parts: [], - metadata: { - time: { - created: Date.now(), - }, - sessionID, - tool: {}, - }, - }; msgs.push(next); let text: TextUIPart | undefined; -- cgit v1.2.3