diff options
Diffstat (limited to 'js/src/tool/fetch.ts')
| -rw-r--r-- | js/src/tool/fetch.ts | 137 |
1 files changed, 0 insertions, 137 deletions
diff --git a/js/src/tool/fetch.ts b/js/src/tool/fetch.ts deleted file mode 100644 index 573e0eec7..000000000 --- a/js/src/tool/fetch.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { z } from "zod"; -import { Tool } from "./tool"; -import { JSDOM } from "jsdom"; -import TurndownService from "turndown"; - -const MAX_RESPONSE_SIZE = 5 * 1024 * 1024; // 5MB -const DEFAULT_TIMEOUT = 30 * 1000; // 30 seconds -const MAX_TIMEOUT = 120 * 1000; // 2 minutes - -const DESCRIPTION = `Fetches content from a URL and returns it in the specified format. - -WHEN TO USE THIS TOOL: -- Use when you need to download content from a URL -- Helpful for retrieving documentation, API responses, or web content -- Useful for getting external information to assist with tasks - -HOW TO USE: -- Provide the URL to fetch content from -- Specify the desired output format (text, markdown, or html) -- Optionally set a timeout for the request - -FEATURES: -- Supports three output formats: text, markdown, and html -- Automatically handles HTTP redirects -- Sets reasonable timeouts to prevent hanging -- Validates input parameters before making requests - -LIMITATIONS: -- Maximum response size is 5MB -- Only supports HTTP and HTTPS protocols -- Cannot handle authentication or cookies -- Some websites may block automated requests - -TIPS: -- Use text format for plain text content or simple API responses -- Use markdown format for content that should be rendered with formatting -- Use html format when you need the raw HTML structure -- Set appropriate timeouts for potentially slow websites`; - -export const Fetch = Tool.define({ - name: "opencode.fetch", - description: DESCRIPTION, - parameters: z.object({ - url: z.string().describe("The URL to fetch content from"), - format: z - .enum(["text", "markdown", "html"]) - .describe( - "The format to return the content in (text, markdown, or html)", - ), - timeout: z - .number() - .min(0) - .max(MAX_TIMEOUT / 1000) - .describe("Optional timeout in seconds (max 120)") - .optional(), - }), - async execute(params, opts) { - // Validate URL - if ( - !params.url.startsWith("http://") && - !params.url.startsWith("https://") - ) { - throw new Error("URL must start with http:// or https://"); - } - - const timeout = Math.min( - (params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, - MAX_TIMEOUT, - ); - - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout); - if (opts?.abortSignal) { - opts.abortSignal.addEventListener("abort", () => controller.abort()); - } - - const response = await fetch(params.url, { - signal: controller.signal, - headers: { - "User-Agent": "opencode/1.0", - }, - }); - - clearTimeout(timeoutId); - - if (!response.ok) { - throw new Error(`Request failed with status code: ${response.status}`); - } - - // Check content length - const contentLength = response.headers.get("content-length"); - if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) { - throw new Error("Response too large (exceeds 5MB limit)"); - } - - const arrayBuffer = await response.arrayBuffer(); - if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) { - throw new Error("Response too large (exceeds 5MB limit)"); - } - - const content = new TextDecoder().decode(arrayBuffer); - const contentType = response.headers.get("content-type") || ""; - - switch (params.format) { - case "text": - if (contentType.includes("text/html")) { - const text = extractTextFromHTML(content); - return { output: text }; - } - return { output: content }; - - case "markdown": - if (contentType.includes("text/html")) { - const markdown = convertHTMLToMarkdown(content); - return { output: markdown }; - } - return { output: "```\n" + content + "\n```" }; - - case "html": - return { output: content }; - - default: - return { output: content }; - } - }, -}); - -function extractTextFromHTML(html: string): string { - const dom = new JSDOM(html); - const text = dom.window.document.body?.textContent || ""; - return text.replace(/\s+/g, " ").trim(); -} - -function convertHTMLToMarkdown(html: string): string { - const turndownService = new TurndownService(); - return turndownService.turndown(html); -} |
