diff options
| -rw-r--r-- | packages/opencode/src/tool/registry.ts | 6 | ||||
| -rw-r--r-- | packages/opencode/src/tool/webfetch.ts | 290 | ||||
| -rw-r--r-- | packages/opencode/test/memory/abort-leak.test.ts | 8 | ||||
| -rw-r--r-- | packages/opencode/test/session/prompt-effect.test.ts | 2 | ||||
| -rw-r--r-- | packages/opencode/test/session/snapshot-tool-race.test.ts | 5 | ||||
| -rw-r--r-- | packages/opencode/test/tool/webfetch.test.ts | 16 |
6 files changed, 172 insertions, 155 deletions
diff --git a/packages/opencode/src/tool/registry.ts b/packages/opencode/src/tool/registry.ts index 716a8ca4e..6d0a6e0cd 100644 --- a/packages/opencode/src/tool/registry.ts +++ b/packages/opencode/src/tool/registry.ts @@ -30,6 +30,7 @@ import { Glob } from "../util/glob" import path from "path" import { pathToFileURL } from "url" import { Effect, Layer, ServiceMap } from "effect" +import { FetchHttpClient, HttpClient } from "effect/unstable/http" import { InstanceState } from "@/effect/instance-state" import { makeRuntime } from "@/effect/run-service" import { Env } from "../env" @@ -84,6 +85,7 @@ export namespace ToolRegistry { | FileTime.Service | Instruction.Service | AppFileSystem.Service + | HttpClient.HttpClient > = Layer.effect( Service, Effect.gen(function* () { @@ -98,6 +100,7 @@ export namespace ToolRegistry { const todo = yield* TodoWriteTool const lsptool = yield* LspTool const plan = yield* PlanExitTool + const webfetch = yield* WebFetchTool const state = yield* InstanceState.make<State>( Effect.fn("ToolRegistry.state")(function* (ctx) { @@ -163,7 +166,7 @@ export namespace ToolRegistry { edit: Tool.init(EditTool), write: Tool.init(WriteTool), task: Tool.init(task), - fetch: Tool.init(WebFetchTool), + fetch: Tool.init(webfetch), todo: Tool.init(todo), search: Tool.init(WebSearchTool), code: Tool.init(CodeSearchTool), @@ -309,6 +312,7 @@ export namespace ToolRegistry { Layer.provide(FileTime.defaultLayer), Layer.provide(Instruction.defaultLayer), Layer.provide(AppFileSystem.defaultLayer), + Layer.provide(FetchHttpClient.layer), ), ) diff --git a/packages/opencode/src/tool/webfetch.ts b/packages/opencode/src/tool/webfetch.ts index 559afd677..515b88e25 100644 --- a/packages/opencode/src/tool/webfetch.ts +++ b/packages/opencode/src/tool/webfetch.ts @@ -1,169 +1,163 @@ import z from "zod" +import { Effect } from "effect" +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" import { Tool } from "./tool" import TurndownService from "turndown" import DESCRIPTION from "./webfetch.txt" -import { abortAfterAny } from "../util/abort" -import { iife } from "@/util/iife" const MAX_RESPONSE_SIZE = 5 * 1024 * 1024 // 5MB const DEFAULT_TIMEOUT = 30 * 1000 // 30 seconds const MAX_TIMEOUT = 120 * 1000 // 2 minutes -export const WebFetchTool = Tool.define("webfetch", { - description: DESCRIPTION, - parameters: z.object({ - url: z.string().describe("The URL to fetch content from"), - format: z - .enum(["text", "markdown", "html"]) - .default("markdown") - .describe("The format to return the content in (text, markdown, or html). Defaults to markdown."), - timeout: z.number().describe("Optional timeout in seconds (max 120)").optional(), - }), - async execute(params, ctx) { - // Validate URL - if (!params.url.startsWith("http://") && !params.url.startsWith("https://")) { - throw new Error("URL must start with http:// or https://") - } - - await ctx.ask({ - permission: "webfetch", - patterns: [params.url], - always: ["*"], - metadata: { - url: params.url, - format: params.format, - timeout: params.timeout, - }, - }) - - const timeout = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT) - - const { signal, clearTimeout } = abortAfterAny(timeout, ctx.abort) - - // Build Accept header based on requested format with q parameters for fallbacks - let acceptHeader = "*/*" - switch (params.format) { - case "markdown": - acceptHeader = "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1" - break - case "text": - acceptHeader = "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1" - break - case "html": - acceptHeader = "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1" - break - default: - acceptHeader = - "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8" - } - const headers = { - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36", - Accept: acceptHeader, - "Accept-Language": "en-US,en;q=0.9", - } - - const response = await iife(async () => { - try { - const initial = await fetch(params.url, { signal, headers }) - - // Retry with honest UA if blocked by Cloudflare bot detection (TLS fingerprint mismatch) - return initial.status === 403 && initial.headers.get("cf-mitigated") === "challenge" - ? await fetch(params.url, { signal, headers: { ...headers, "User-Agent": "opencode" } }) - : initial - } finally { - clearTimeout() - } - }) - - if (!response.ok) { - throw new Error(`Request failed with status code: ${response.status}`) - } - - // Check content length - const contentLength = response.headers.get("content-length") - if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) { - throw new Error("Response too large (exceeds 5MB limit)") - } +const parameters = z.object({ + url: z.string().describe("The URL to fetch content from"), + format: z + .enum(["text", "markdown", "html"]) + .default("markdown") + .describe("The format to return the content in (text, markdown, or html). Defaults to markdown."), + timeout: z.number().describe("Optional timeout in seconds (max 120)").optional(), +}) - const arrayBuffer = await response.arrayBuffer() - if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) { - throw new Error("Response too large (exceeds 5MB limit)") - } +export const WebFetchTool = Tool.defineEffect( + "webfetch", + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient + const httpOk = HttpClient.filterStatusOk(http) + + return { + description: DESCRIPTION, + parameters, + execute: (params: z.infer<typeof parameters>, ctx: Tool.Context) => + Effect.gen(function* () { + if (!params.url.startsWith("http://") && !params.url.startsWith("https://")) { + throw new Error("URL must start with http:// or https://") + } - const contentType = response.headers.get("content-type") || "" - const mime = contentType.split(";")[0]?.trim().toLowerCase() || "" - const title = `${params.url} (${contentType})` - - // Check if response is an image - const isImage = mime.startsWith("image/") && mime !== "image/svg+xml" && mime !== "image/vnd.fastbidsheet" - - if (isImage) { - const base64Content = Buffer.from(arrayBuffer).toString("base64") - return { - title, - output: "Image fetched successfully", - metadata: {}, - attachments: [ - { - type: "file", - mime, - url: `data:${mime};base64,${base64Content}`, - }, - ], - } - } + yield* Effect.promise(() => + ctx.ask({ + permission: "webfetch", + patterns: [params.url], + always: ["*"], + metadata: { + url: params.url, + format: params.format, + timeout: params.timeout, + }, + }), + ) + + const timeout = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT) + + // Build Accept header based on requested format with q parameters for fallbacks + let acceptHeader = "*/*" + switch (params.format) { + case "markdown": + acceptHeader = + "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1" + break + case "text": + acceptHeader = "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1" + break + case "html": + acceptHeader = + "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1" + break + default: + acceptHeader = + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8" + } + const headers = { + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36", + Accept: acceptHeader, + "Accept-Language": "en-US,en;q=0.9", + } - const content = new TextDecoder().decode(arrayBuffer) - - // Handle content based on requested format and actual content type - switch (params.format) { - case "markdown": - if (contentType.includes("text/html")) { - const markdown = convertHTMLToMarkdown(content) - return { - output: markdown, - title, - metadata: {}, + const request = HttpClientRequest.get(params.url).pipe(HttpClientRequest.setHeaders(headers)) + + // Retry with honest UA if blocked by Cloudflare bot detection (TLS fingerprint mismatch) + const response = yield* httpOk.execute(request).pipe( + Effect.catchIf( + (err) => + err.reason._tag === "StatusCodeError" && + err.reason.response.status === 403 && + err.reason.response.headers["cf-mitigated"] === "challenge", + () => + httpOk.execute( + HttpClientRequest.get(params.url).pipe( + HttpClientRequest.setHeaders({ ...headers, "User-Agent": "opencode" }), + ), + ), + ), + Effect.timeoutOrElse({ duration: timeout, orElse: () => Effect.die(new Error("Request timed out")) }), + ) + + // Check content length + const contentLength = response.headers["content-length"] + if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) { + throw new Error("Response too large (exceeds 5MB limit)") } - } - return { - output: content, - title, - metadata: {}, - } - case "text": - if (contentType.includes("text/html")) { - const text = await extractTextFromHTML(content) - return { - output: text, - title, - metadata: {}, + const arrayBuffer = yield* response.arrayBuffer + if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) { + throw new Error("Response too large (exceeds 5MB limit)") } - } - return { - output: content, - title, - metadata: {}, - } - case "html": - return { - output: content, - title, - metadata: {}, - } + const contentType = response.headers["content-type"] || "" + const mime = contentType.split(";")[0]?.trim().toLowerCase() || "" + const title = `${params.url} (${contentType})` + + // Check if response is an image + const isImage = mime.startsWith("image/") && mime !== "image/svg+xml" && mime !== "image/vnd.fastbidsheet" + + if (isImage) { + const base64Content = Buffer.from(arrayBuffer).toString("base64") + return { + title, + output: "Image fetched successfully", + metadata: {}, + attachments: [ + { + type: "file" as const, + mime, + url: `data:${mime};base64,${base64Content}`, + }, + ], + } + } - default: - return { - output: content, - title, - metadata: {}, - } + const content = new TextDecoder().decode(arrayBuffer) + + // Handle content based on requested format and actual content type + switch (params.format) { + case "markdown": + if (contentType.includes("text/html")) { + const markdown = convertHTMLToMarkdown(content) + return { + output: markdown, + title, + metadata: {}, + } + } + return { output: content, title, metadata: {} } + + case "text": + if (contentType.includes("text/html")) { + const text = yield* Effect.promise(() => extractTextFromHTML(content)) + return { output: text, title, metadata: {} } + } + return { output: content, title, metadata: {} } + + case "html": + return { output: content, title, metadata: {} } + + default: + return { output: content, title, metadata: {} } + } + }).pipe(Effect.runPromise), } - }, -}) + }), +) async function extractTextFromHTML(html: string) { let text = "" diff --git a/packages/opencode/test/memory/abort-leak.test.ts b/packages/opencode/test/memory/abort-leak.test.ts index eebb651a5..2d0180783 100644 --- a/packages/opencode/test/memory/abort-leak.test.ts +++ b/packages/opencode/test/memory/abort-leak.test.ts @@ -1,5 +1,7 @@ import { describe, test, expect } from "bun:test" import path from "path" +import { Effect } from "effect" +import { FetchHttpClient } from "effect/unstable/http" import { Instance } from "../../src/project/instance" import { WebFetchTool } from "../../src/tool/webfetch" import { SessionID, MessageID } from "../../src/session/schema" @@ -30,7 +32,11 @@ describe("memory: abort controller leak", () => { await Instance.provide({ directory: projectRoot, fn: async () => { - const tool = await WebFetchTool.init() + const tool = await WebFetchTool.pipe( + Effect.flatMap((info) => Effect.promise(() => info.init())), + Effect.provide(FetchHttpClient.layer), + Effect.runPromise, + ) // Warm up await tool.execute({ url: "https://example.com", format: "text" }, ctx).catch(() => {}) diff --git a/packages/opencode/test/session/prompt-effect.test.ts b/packages/opencode/test/session/prompt-effect.test.ts index 215f6668c..ef8512bad 100644 --- a/packages/opencode/test/session/prompt-effect.test.ts +++ b/packages/opencode/test/session/prompt-effect.test.ts @@ -1,6 +1,7 @@ import { NodeFileSystem } from "@effect/platform-node" import { expect } from "bun:test" import { Cause, Effect, Exit, Fiber, Layer } from "effect" +import { FetchHttpClient } from "effect/unstable/http" import path from "path" import z from "zod" import { Agent as AgentSvc } from "../../src/agent/agent" @@ -169,6 +170,7 @@ function makeHttp() { const todo = Todo.layer.pipe(Layer.provideMerge(deps)) const registry = ToolRegistry.layer.pipe( Layer.provide(Skill.defaultLayer), + Layer.provide(FetchHttpClient.layer), Layer.provideMerge(todo), Layer.provideMerge(question), Layer.provideMerge(deps), diff --git a/packages/opencode/test/session/snapshot-tool-race.test.ts b/packages/opencode/test/session/snapshot-tool-race.test.ts index ae67983bf..b8ee6b284 100644 --- a/packages/opencode/test/session/snapshot-tool-race.test.ts +++ b/packages/opencode/test/session/snapshot-tool-race.test.ts @@ -12,7 +12,8 @@ * tools internally during multi-step processing before emitting events. */ import { expect } from "bun:test" -import { Effect } from "effect" +import { Effect, Layer } from "effect" +import { FetchHttpClient } from "effect/unstable/http" import fs from "fs/promises" import path from "path" import { Session } from "../../src/session" @@ -28,7 +29,6 @@ import { TestLLMServer } from "../lib/llm-server" // Same layer setup as prompt-effect.test.ts import { NodeFileSystem } from "@effect/platform-node" -import { Layer } from "effect" import { Agent as AgentSvc } from "../../src/agent/agent" import { Bus } from "../../src/bus" import { Command } from "../../src/command" @@ -134,6 +134,7 @@ function makeHttp() { const todo = Todo.layer.pipe(Layer.provideMerge(deps)) const registry = ToolRegistry.layer.pipe( Layer.provide(Skill.defaultLayer), + Layer.provide(FetchHttpClient.layer), Layer.provideMerge(todo), Layer.provideMerge(question), Layer.provideMerge(deps), diff --git a/packages/opencode/test/tool/webfetch.test.ts b/packages/opencode/test/tool/webfetch.test.ts index 00e9dcc96..a26be24ae 100644 --- a/packages/opencode/test/tool/webfetch.test.ts +++ b/packages/opencode/test/tool/webfetch.test.ts @@ -1,5 +1,7 @@ import { describe, expect, test } from "bun:test" import path from "path" +import { Effect } from "effect" +import { FetchHttpClient } from "effect/unstable/http" import { Instance } from "../../src/project/instance" import { WebFetchTool } from "../../src/tool/webfetch" import { SessionID, MessageID } from "../../src/session/schema" @@ -22,6 +24,14 @@ async function withFetch(fetch: (req: Request) => Response | Promise<Response>, await fn(server.url) } +function initTool() { + return WebFetchTool.pipe( + Effect.flatMap((info) => Effect.promise(() => info.init())), + Effect.provide(FetchHttpClient.layer), + Effect.runPromise, + ) +} + describe("tool.webfetch", () => { test("returns image responses as file attachments", async () => { const bytes = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]) @@ -31,7 +41,7 @@ describe("tool.webfetch", () => { await Instance.provide({ directory: projectRoot, fn: async () => { - const webfetch = await WebFetchTool.init() + const webfetch = await initTool() const result = await webfetch.execute( { url: new URL("/image.png", url).toString(), format: "markdown" }, ctx, @@ -63,7 +73,7 @@ describe("tool.webfetch", () => { await Instance.provide({ directory: projectRoot, fn: async () => { - const webfetch = await WebFetchTool.init() + const webfetch = await initTool() const result = await webfetch.execute({ url: new URL("/image.svg", url).toString(), format: "html" }, ctx) expect(result.output).toContain("<svg") expect(result.attachments).toBeUndefined() @@ -84,7 +94,7 @@ describe("tool.webfetch", () => { await Instance.provide({ directory: projectRoot, fn: async () => { - const webfetch = await WebFetchTool.init() + const webfetch = await initTool() const result = await webfetch.execute({ url: new URL("/file.txt", url).toString(), format: "text" }, ctx) expect(result.output).toBe("hello from webfetch") expect(result.attachments).toBeUndefined() |