1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
import { z } from "zod";
import { Tool } from "./tool";
import { JSDOM } from "jsdom";
import TurndownService from "turndown";
const MAX_RESPONSE_SIZE = 5 * 1024 * 1024; // 5MB
const DEFAULT_TIMEOUT = 30 * 1000; // 30 seconds
const MAX_TIMEOUT = 120 * 1000; // 2 minutes
const DESCRIPTION = `Fetches content from a URL and returns it in the specified format.
WHEN TO USE THIS TOOL:
- Use when you need to download content from a URL
- Helpful for retrieving documentation, API responses, or web content
- Useful for getting external information to assist with tasks
HOW TO USE:
- Provide the URL to fetch content from
- Specify the desired output format (text, markdown, or html)
- Optionally set a timeout for the request
FEATURES:
- Supports three output formats: text, markdown, and html
- Automatically handles HTTP redirects
- Sets reasonable timeouts to prevent hanging
- Validates input parameters before making requests
LIMITATIONS:
- Maximum response size is 5MB
- Only supports HTTP and HTTPS protocols
- Cannot handle authentication or cookies
- Some websites may block automated requests
TIPS:
- Use text format for plain text content or simple API responses
- Use markdown format for content that should be rendered with formatting
- Use html format when you need the raw HTML structure
- Set appropriate timeouts for potentially slow websites`;
export const Fetch = Tool.define({
name: "fetch",
description: DESCRIPTION,
parameters: z.object({
url: z.string().describe("The URL to fetch content from"),
format: z
.enum(["text", "markdown", "html"])
.describe(
"The format to return the content in (text, markdown, or html)",
),
timeout: z
.number()
.min(0)
.max(MAX_TIMEOUT / 1000)
.describe("Optional timeout in seconds (max 120)")
.optional(),
}),
async execute(params, opts) {
// Validate URL
if (
!params.url.startsWith("http://") &&
!params.url.startsWith("https://")
) {
throw new Error("URL must start with http:// or https://");
}
const timeout = Math.min(
(params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000,
MAX_TIMEOUT,
);
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
if (opts?.abortSignal) {
opts.abortSignal.addEventListener("abort", () => controller.abort());
}
const response = await fetch(params.url, {
signal: controller.signal,
headers: {
"User-Agent": "opencode/1.0",
},
});
clearTimeout(timeoutId);
if (!response.ok) {
throw new Error(`Request failed with status code: ${response.status}`);
}
// Check content length
const contentLength = response.headers.get("content-length");
if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
throw new Error("Response too large (exceeds 5MB limit)");
}
const arrayBuffer = await response.arrayBuffer();
if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
throw new Error("Response too large (exceeds 5MB limit)");
}
const content = new TextDecoder().decode(arrayBuffer);
const contentType = response.headers.get("content-type") || "";
switch (params.format) {
case "text":
if (contentType.includes("text/html")) {
const text = extractTextFromHTML(content);
return { output: text };
}
return { output: content };
case "markdown":
if (contentType.includes("text/html")) {
const markdown = convertHTMLToMarkdown(content);
return { output: markdown };
}
return { output: "```\n" + content + "\n```" };
case "html":
return { output: content };
default:
return { output: content };
}
},
});
function extractTextFromHTML(html: string): string {
const dom = new JSDOM(html);
const text = dom.window.document.body?.textContent || "";
return text.replace(/\s+/g, " ").trim();
}
function convertHTMLToMarkdown(html: string): string {
const turndownService = new TurndownService();
return turndownService.turndown(html);
}
|
