diff options
| -rw-r--r-- | packages/opencode/package.json | 1 | ||||
| -rw-r--r-- | packages/opencode/src/effect/instances.ts | 3 | ||||
| -rw-r--r-- | packages/opencode/src/skill/discovery.ts | 190 | ||||
| -rw-r--r-- | packages/opencode/src/skill/skill.ts | 333 | ||||
| -rw-r--r-- | packages/opencode/test/skill/discovery.test.ts | 30 |
5 files changed, 318 insertions, 239 deletions
diff --git a/packages/opencode/package.json b/packages/opencode/package.json index c462b1761..0463cc6d2 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -95,6 +95,7 @@ "@openrouter/ai-sdk-provider": "1.5.4", "@opentui/core": "0.1.87", "@opentui/solid": "0.1.87", + "@effect/platform-node": "4.0.0-beta.31", "@parcel/watcher": "2.5.1", "@pierre/diffs": "catalog:", "@solid-primitives/event-bus": "1.1.2", diff --git a/packages/opencode/src/effect/instances.ts b/packages/opencode/src/effect/instances.ts index 240f8ee66..eabf19868 100644 --- a/packages/opencode/src/effect/instances.ts +++ b/packages/opencode/src/effect/instances.ts @@ -9,6 +9,7 @@ import { VcsService } from "@/project/vcs" import { FileTimeService } from "@/file/time" import { FormatService } from "@/format" import { FileService } from "@/file" +import { SkillService } from "@/skill/skill" import { Instance } from "@/project/instance" export { InstanceContext } from "./instance-context" @@ -22,6 +23,7 @@ export type InstanceServices = | FileTimeService | FormatService | FileService + | SkillService function lookup(directory: string) { const project = Instance.project @@ -35,6 +37,7 @@ function lookup(directory: string) { Layer.fresh(FileTimeService.layer).pipe(Layer.orDie), Layer.fresh(FormatService.layer), Layer.fresh(FileService.layer), + Layer.fresh(SkillService.layer), ).pipe(Layer.provide(ctx)) } diff --git a/packages/opencode/src/skill/discovery.ts b/packages/opencode/src/skill/discovery.ts index 846002cda..fe03dccef 100644 --- a/packages/opencode/src/skill/discovery.ts +++ b/packages/opencode/src/skill/discovery.ts @@ -1,98 +1,118 @@ -import path from "path" -import { mkdir } from "fs/promises" -import { Log } from "../util/log" +import { NodeFileSystem, NodePath } from "@effect/platform-node" +import { Effect, FileSystem, Layer, Path, Schema, ServiceMap } from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" import { Global } from "../global" -import { Filesystem } from "../util/filesystem" +import { Log } from "../util/log" +import { withTransientReadRetry } from "@/util/effect-http-client" -export namespace Discovery { - const log = Log.create({ service: "skill-discovery" }) +class IndexSkill extends Schema.Class<IndexSkill>("IndexSkill")({ + name: Schema.String, + files: Schema.Array(Schema.String), +}) {} - type Index = { - skills: Array<{ - name: string - description: string - files: string[] - }> - } +class Index extends Schema.Class<Index>("Index")({ + skills: Schema.Array(IndexSkill), +}) {} - export function dir() { - return path.join(Global.Path.cache, "skills") - } +const skillConcurrency = 4 +const fileConcurrency = 8 - async function get(url: string, dest: string): Promise<boolean> { - if (await Filesystem.exists(dest)) return true - return fetch(url) - .then(async (response) => { - if (!response.ok) { - log.error("failed to download", { url, status: response.status }) - return false - } - if (response.body) await Filesystem.writeStream(dest, response.body) - return true - }) - .catch((err) => { - log.error("failed to download", { url, err }) - return false - }) +export namespace DiscoveryService { + export interface Service { + readonly pull: (url: string) => Effect.Effect<string[]> } +} - export async function pull(url: string): Promise<string[]> { - const result: string[] = [] - const base = url.endsWith("/") ? url : `${url}/` - const index = new URL("index.json", base).href - const cache = dir() - const host = base.slice(0, -1) - - log.info("fetching index", { url: index }) - const data = await fetch(index) - .then(async (response) => { - if (!response.ok) { - log.error("failed to fetch index", { url: index, status: response.status }) - return undefined - } - return response - .json() - .then((json) => json as Index) - .catch((err) => { - log.error("failed to parse index", { url: index, err }) - return undefined - }) - }) - .catch((err) => { - log.error("failed to fetch index", { url: index, err }) - return undefined +export class DiscoveryService extends ServiceMap.Service<DiscoveryService, DiscoveryService.Service>()( + "@opencode/SkillDiscovery", +) { + static readonly layer = Layer.effect( + DiscoveryService, + Effect.gen(function* () { + const log = Log.create({ service: "skill-discovery" }) + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + const http = HttpClient.filterStatusOk(withTransientReadRetry(yield* HttpClient.HttpClient)) + const cache = path.join(Global.Path.cache, "skills") + + const download = Effect.fn("DiscoveryService.download")(function* (url: string, dest: string) { + if (yield* fs.exists(dest).pipe(Effect.orDie)) return true + + return yield* HttpClientRequest.get(url).pipe( + http.execute, + Effect.flatMap((res) => res.arrayBuffer), + Effect.flatMap((body) => + fs + .makeDirectory(path.dirname(dest), { recursive: true }) + .pipe(Effect.flatMap(() => fs.writeFile(dest, new Uint8Array(body)))), + ), + Effect.as(true), + Effect.catch((err) => + Effect.sync(() => { + log.error("failed to download", { url, err }) + return false + }), + ), + ) }) - if (!data?.skills || !Array.isArray(data.skills)) { - log.warn("invalid index format", { url: index }) - return result - } - - const list = data.skills.filter((skill) => { - if (!skill?.name || !Array.isArray(skill.files)) { - log.warn("invalid skill entry", { url: index, skill }) - return false - } - return true - }) - - await Promise.all( - list.map(async (skill) => { - const root = path.join(cache, skill.name) - await Promise.all( - skill.files.map(async (file) => { - const link = new URL(file, `${host}/${skill.name}/`).href - const dest = path.join(root, file) - await mkdir(path.dirname(dest), { recursive: true }) - await get(link, dest) - }), + const pull: DiscoveryService.Service["pull"] = Effect.fn("DiscoveryService.pull")(function* (url: string) { + const base = url.endsWith("/") ? url : `${url}/` + const index = new URL("index.json", base).href + const host = base.slice(0, -1) + + log.info("fetching index", { url: index }) + + const data = yield* HttpClientRequest.get(index).pipe( + HttpClientRequest.acceptJson, + http.execute, + Effect.flatMap(HttpClientResponse.schemaBodyJson(Index)), + Effect.catch((err) => + Effect.sync(() => { + log.error("failed to fetch index", { url: index, err }) + return null + }), + ), ) - const md = path.join(root, "SKILL.md") - if (await Filesystem.exists(md)) result.push(root) - }), - ) + if (!data) return [] - return result - } + const list = data.skills.filter((skill) => { + if (!skill.files.includes("SKILL.md")) { + log.warn("skill entry missing SKILL.md", { url: index, skill: skill.name }) + return false + } + return true + }) + + const dirs = yield* Effect.forEach( + list, + (skill) => + Effect.gen(function* () { + const root = path.join(cache, skill.name) + + yield* Effect.forEach( + skill.files, + (file) => download(new URL(file, `${host}/${skill.name}/`).href, path.join(root, file)), + { concurrency: fileConcurrency }, + ) + + const md = path.join(root, "SKILL.md") + return (yield* fs.exists(md).pipe(Effect.orDie)) ? root : null + }), + { concurrency: skillConcurrency }, + ) + + return dirs.filter((dir): dir is string => dir !== null) + }) + + return DiscoveryService.of({ pull }) + }), + ) + + static readonly defaultLayer = DiscoveryService.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(NodeFileSystem.layer), + Layer.provide(NodePath.layer), + ) } diff --git a/packages/opencode/src/skill/skill.ts b/packages/opencode/src/skill/skill.ts index fa984b3e1..3a544d90a 100644 --- a/packages/opencode/src/skill/skill.ts +++ b/packages/opencode/src/skill/skill.ts @@ -10,15 +10,25 @@ import { Global } from "@/global" import { Filesystem } from "@/util/filesystem" import { Flag } from "@/flag/flag" import { Bus } from "@/bus" -import { Session } from "@/session" -import { Discovery } from "./discovery" +import { DiscoveryService } from "./discovery" import { Glob } from "../util/glob" import { pathToFileURL } from "url" import type { Agent } from "@/agent/agent" import { PermissionNext } from "@/permission/next" +import { InstanceContext } from "@/effect/instance-context" +import { Effect, Layer, ServiceMap } from "effect" +import { runPromiseInstance } from "@/effect/runtime" + +const log = Log.create({ service: "skill" }) + +// External skill directories to search for (project-level and global) +// These follow the directory layout used by Claude Code and other agents. +const EXTERNAL_DIRS = [".claude", ".agents"] +const EXTERNAL_SKILL_PATTERN = "skills/**/SKILL.md" +const OPENCODE_SKILL_PATTERN = "{skill,skills}/**/SKILL.md" +const SKILL_PATTERN = "**/SKILL.md" export namespace Skill { - const log = Log.create({ service: "skill" }) export const Info = z.object({ name: z.string(), description: z.string(), @@ -45,155 +55,20 @@ export namespace Skill { }), ) - // External skill directories to search for (project-level and global) - // These follow the directory layout used by Claude Code and other agents. - const EXTERNAL_DIRS = [".claude", ".agents"] - const EXTERNAL_SKILL_PATTERN = "skills/**/SKILL.md" - const OPENCODE_SKILL_PATTERN = "{skill,skills}/**/SKILL.md" - const SKILL_PATTERN = "**/SKILL.md" - - export const state = Instance.state(async () => { - const skills: Record<string, Info> = {} - const dirs = new Set<string>() - - const addSkill = async (match: string) => { - const md = await ConfigMarkdown.parse(match).catch((err) => { - const message = ConfigMarkdown.FrontmatterError.isInstance(err) - ? err.data.message - : `Failed to parse skill ${match}` - Bus.publish(Session.Event.Error, { error: new NamedError.Unknown({ message }).toObject() }) - log.error("failed to load skill", { skill: match, err }) - return undefined - }) - - if (!md) return - - const parsed = Info.pick({ name: true, description: true }).safeParse(md.data) - if (!parsed.success) return - - // Warn on duplicate skill names - if (skills[parsed.data.name]) { - log.warn("duplicate skill name", { - name: parsed.data.name, - existing: skills[parsed.data.name].location, - duplicate: match, - }) - } - - dirs.add(path.dirname(match)) - - skills[parsed.data.name] = { - name: parsed.data.name, - description: parsed.data.description, - location: match, - content: md.content, - } - } - - const scanExternal = async (root: string, scope: "global" | "project") => { - return Glob.scan(EXTERNAL_SKILL_PATTERN, { - cwd: root, - absolute: true, - include: "file", - dot: true, - symlink: true, - }) - .then((matches) => Promise.all(matches.map(addSkill))) - .catch((error) => { - log.error(`failed to scan ${scope} skills`, { dir: root, error }) - }) - } - - // Scan external skill directories (.claude/skills/, .agents/skills/, etc.) - // Load global (home) first, then project-level (so project-level overwrites) - if (!Flag.OPENCODE_DISABLE_EXTERNAL_SKILLS) { - for (const dir of EXTERNAL_DIRS) { - const root = path.join(Global.Path.home, dir) - if (!(await Filesystem.isDir(root))) continue - await scanExternal(root, "global") - } - - for await (const root of Filesystem.up({ - targets: EXTERNAL_DIRS, - start: Instance.directory, - stop: Instance.worktree, - })) { - await scanExternal(root, "project") - } - } - - // Scan .opencode/skill/ directories - for (const dir of await Config.directories()) { - const matches = await Glob.scan(OPENCODE_SKILL_PATTERN, { - cwd: dir, - absolute: true, - include: "file", - symlink: true, - }) - for (const match of matches) { - await addSkill(match) - } - } - - // Scan additional skill paths from config - const config = await Config.get() - for (const skillPath of config.skills?.paths ?? []) { - const expanded = skillPath.startsWith("~/") ? path.join(os.homedir(), skillPath.slice(2)) : skillPath - const resolved = path.isAbsolute(expanded) ? expanded : path.join(Instance.directory, expanded) - if (!(await Filesystem.isDir(resolved))) { - log.warn("skill path not found", { path: resolved }) - continue - } - const matches = await Glob.scan(SKILL_PATTERN, { - cwd: resolved, - absolute: true, - include: "file", - symlink: true, - }) - for (const match of matches) { - await addSkill(match) - } - } - - // Download and load skills from URLs - for (const url of config.skills?.urls ?? []) { - const list = await Discovery.pull(url) - for (const dir of list) { - dirs.add(dir) - const matches = await Glob.scan(SKILL_PATTERN, { - cwd: dir, - absolute: true, - include: "file", - symlink: true, - }) - for (const match of matches) { - await addSkill(match) - } - } - } - - return { - skills, - dirs: Array.from(dirs), - } - }) - export async function get(name: string) { - return state().then((x) => x.skills[name]) + return runPromiseInstance(SkillService.use((s) => s.get(name))) } export async function all() { - return state().then((x) => Object.values(x.skills)) + return runPromiseInstance(SkillService.use((s) => s.all())) } export async function dirs() { - return state().then((x) => x.dirs) + return runPromiseInstance(SkillService.use((s) => s.dirs())) } export async function available(agent?: Agent.Info) { - const list = await all() - if (!agent) return list - return list.filter((skill) => PermissionNext.evaluate("skill", skill.name, agent.permission).action !== "deny") + return runPromiseInstance(SkillService.use((s) => s.available(agent))) } export function fmt(list: Info[], opts: { verbose: boolean }) { @@ -216,3 +91,177 @@ export namespace Skill { return ["## Available Skills", ...list.flatMap((skill) => `- **${skill.name}**: ${skill.description}`)].join("\n") } } + +export namespace SkillService { + export interface Service { + readonly get: (name: string) => Effect.Effect<Skill.Info | undefined> + readonly all: () => Effect.Effect<Skill.Info[]> + readonly dirs: () => Effect.Effect<string[]> + readonly available: (agent?: Agent.Info) => Effect.Effect<Skill.Info[]> + } +} + +export class SkillService extends ServiceMap.Service<SkillService, SkillService.Service>()("@opencode/Skill") { + static readonly layer = Layer.effect( + SkillService, + Effect.gen(function* () { + const instance = yield* InstanceContext + const discovery = yield* DiscoveryService + + const skills: Record<string, Skill.Info> = {} + const skillDirs = new Set<string>() + let task: Promise<void> | undefined + + const addSkill = async (match: string) => { + const md = await ConfigMarkdown.parse(match).catch(async (err) => { + const message = ConfigMarkdown.FrontmatterError.isInstance(err) + ? err.data.message + : `Failed to parse skill ${match}` + const { Session } = await import("@/session") + Bus.publish(Session.Event.Error, { error: new NamedError.Unknown({ message }).toObject() }) + log.error("failed to load skill", { skill: match, err }) + return undefined + }) + + if (!md) return + + const parsed = Skill.Info.pick({ name: true, description: true }).safeParse(md.data) + if (!parsed.success) return + + // Warn on duplicate skill names + if (skills[parsed.data.name]) { + log.warn("duplicate skill name", { + name: parsed.data.name, + existing: skills[parsed.data.name].location, + duplicate: match, + }) + } + + skillDirs.add(path.dirname(match)) + + skills[parsed.data.name] = { + name: parsed.data.name, + description: parsed.data.description, + location: match, + content: md.content, + } + } + + const scanExternal = async (root: string, scope: "global" | "project") => { + return Glob.scan(EXTERNAL_SKILL_PATTERN, { + cwd: root, + absolute: true, + include: "file", + dot: true, + symlink: true, + }) + .then((matches) => Promise.all(matches.map(addSkill))) + .catch((error) => { + log.error(`failed to scan ${scope} skills`, { dir: root, error }) + }) + } + + function ensureScanned() { + if (task) return task + task = (async () => { + // Scan external skill directories (.claude/skills/, .agents/skills/, etc.) + // Load global (home) first, then project-level (so project-level overwrites) + if (!Flag.OPENCODE_DISABLE_EXTERNAL_SKILLS) { + for (const dir of EXTERNAL_DIRS) { + const root = path.join(Global.Path.home, dir) + if (!(await Filesystem.isDir(root))) continue + await scanExternal(root, "global") + } + + for await (const root of Filesystem.up({ + targets: EXTERNAL_DIRS, + start: instance.directory, + stop: instance.project.worktree, + })) { + await scanExternal(root, "project") + } + } + + // Scan .opencode/skill/ directories + for (const dir of await Config.directories()) { + const matches = await Glob.scan(OPENCODE_SKILL_PATTERN, { + cwd: dir, + absolute: true, + include: "file", + symlink: true, + }) + for (const match of matches) { + await addSkill(match) + } + } + + // Scan additional skill paths from config + const config = await Config.get() + for (const skillPath of config.skills?.paths ?? []) { + const expanded = skillPath.startsWith("~/") ? path.join(os.homedir(), skillPath.slice(2)) : skillPath + const resolved = path.isAbsolute(expanded) ? expanded : path.join(instance.directory, expanded) + if (!(await Filesystem.isDir(resolved))) { + log.warn("skill path not found", { path: resolved }) + continue + } + const matches = await Glob.scan(SKILL_PATTERN, { + cwd: resolved, + absolute: true, + include: "file", + symlink: true, + }) + for (const match of matches) { + await addSkill(match) + } + } + + // Download and load skills from URLs + for (const url of config.skills?.urls ?? []) { + const list = await Effect.runPromise(discovery.pull(url)) + for (const dir of list) { + skillDirs.add(dir) + const matches = await Glob.scan(SKILL_PATTERN, { + cwd: dir, + absolute: true, + include: "file", + symlink: true, + }) + for (const match of matches) { + await addSkill(match) + } + } + } + + log.info("init", { count: Object.keys(skills).length }) + })().catch((err) => { + task = undefined + throw err + }) + return task + } + + return SkillService.of({ + get: Effect.fn("SkillService.get")(function* (name: string) { + yield* Effect.promise(() => ensureScanned()) + return skills[name] + }), + all: Effect.fn("SkillService.all")(function* () { + yield* Effect.promise(() => ensureScanned()) + return Object.values(skills) + }), + dirs: Effect.fn("SkillService.dirs")(function* () { + yield* Effect.promise(() => ensureScanned()) + return Array.from(skillDirs) + }), + available: Effect.fn("SkillService.available")(function* (agent?: Agent.Info) { + yield* Effect.promise(() => ensureScanned()) + const list = Object.values(skills) + if (!agent) return list + return list.filter( + (skill) => PermissionNext.evaluate("skill", skill.name, agent.permission).action !== "deny", + ) + }), + }) + }), + ).pipe(Layer.provide(DiscoveryService.defaultLayer)) +} diff --git a/packages/opencode/test/skill/discovery.test.ts b/packages/opencode/test/skill/discovery.test.ts index 5664fa32b..5cbb3ada0 100644 --- a/packages/opencode/test/skill/discovery.test.ts +++ b/packages/opencode/test/skill/discovery.test.ts @@ -1,5 +1,7 @@ import { describe, test, expect, beforeAll, afterAll } from "bun:test" -import { Discovery } from "../../src/skill/discovery" +import { Effect } from "effect" +import { DiscoveryService } from "../../src/skill/discovery" +import { Global } from "../../src/global" import { Filesystem } from "../../src/util/filesystem" import { rm } from "fs/promises" import path from "path" @@ -9,9 +11,10 @@ let server: ReturnType<typeof Bun.serve> let downloadCount = 0 const fixturePath = path.join(import.meta.dir, "../fixture/skills") +const cacheDir = path.join(Global.Path.cache, "skills") beforeAll(async () => { - await rm(Discovery.dir(), { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) server = Bun.serve({ port: 0, @@ -40,22 +43,25 @@ beforeAll(async () => { afterAll(async () => { server?.stop() - await rm(Discovery.dir(), { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) }) describe("Discovery.pull", () => { + const pull = (url: string) => + Effect.runPromise(DiscoveryService.use((s) => s.pull(url)).pipe(Effect.provide(DiscoveryService.defaultLayer))) + test("downloads skills from cloudflare url", async () => { - const dirs = await Discovery.pull(CLOUDFLARE_SKILLS_URL) + const dirs = await pull(CLOUDFLARE_SKILLS_URL) expect(dirs.length).toBeGreaterThan(0) for (const dir of dirs) { - expect(dir).toStartWith(Discovery.dir()) + expect(dir).toStartWith(cacheDir) const md = path.join(dir, "SKILL.md") expect(await Filesystem.exists(md)).toBe(true) } }) test("url without trailing slash works", async () => { - const dirs = await Discovery.pull(CLOUDFLARE_SKILLS_URL.replace(/\/$/, "")) + const dirs = await pull(CLOUDFLARE_SKILLS_URL.replace(/\/$/, "")) expect(dirs.length).toBeGreaterThan(0) for (const dir of dirs) { const md = path.join(dir, "SKILL.md") @@ -64,18 +70,18 @@ describe("Discovery.pull", () => { }) test("returns empty array for invalid url", async () => { - const dirs = await Discovery.pull(`http://localhost:${server.port}/invalid-url/`) + const dirs = await pull(`http://localhost:${server.port}/invalid-url/`) expect(dirs).toEqual([]) }) test("returns empty array for non-json response", async () => { // any url not explicitly handled in server returns 404 text "Not Found" - const dirs = await Discovery.pull(`http://localhost:${server.port}/some-other-path/`) + const dirs = await pull(`http://localhost:${server.port}/some-other-path/`) expect(dirs).toEqual([]) }) test("downloads reference files alongside SKILL.md", async () => { - const dirs = await Discovery.pull(CLOUDFLARE_SKILLS_URL) + const dirs = await pull(CLOUDFLARE_SKILLS_URL) // find a skill dir that should have reference files (e.g. agents-sdk) const agentsSdk = dirs.find((d) => d.endsWith(path.sep + "agents-sdk")) expect(agentsSdk).toBeDefined() @@ -90,17 +96,17 @@ describe("Discovery.pull", () => { test("caches downloaded files on second pull", async () => { // clear dir and downloadCount - await rm(Discovery.dir(), { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) downloadCount = 0 // first pull to populate cache - const first = await Discovery.pull(CLOUDFLARE_SKILLS_URL) + const first = await pull(CLOUDFLARE_SKILLS_URL) expect(first.length).toBeGreaterThan(0) const firstCount = downloadCount expect(firstCount).toBeGreaterThan(0) // second pull should return same results from cache - const second = await Discovery.pull(CLOUDFLARE_SKILLS_URL) + const second = await pull(CLOUDFLARE_SKILLS_URL) expect(second.length).toBe(first.length) expect(second.sort()).toEqual(first.sort()) |