diff options
Diffstat (limited to 'packages/tool-web-search')
| -rw-r--r-- | packages/tool-web-search/package.json | 18 | ||||
| -rw-r--r-- | packages/tool-web-search/src/client.test.ts | 356 | ||||
| -rw-r--r-- | packages/tool-web-search/src/client.ts | 368 | ||||
| -rw-r--r-- | packages/tool-web-search/src/extension.test.ts | 162 | ||||
| -rw-r--r-- | packages/tool-web-search/src/extension.ts | 22 | ||||
| -rw-r--r-- | packages/tool-web-search/src/format.test.ts | 126 | ||||
| -rw-r--r-- | packages/tool-web-search/src/format.ts | 116 | ||||
| -rw-r--r-- | packages/tool-web-search/src/index.ts | 64 | ||||
| -rw-r--r-- | packages/tool-web-search/src/tool.ts | 216 | ||||
| -rw-r--r-- | packages/tool-web-search/src/validate.test.ts | 150 | ||||
| -rw-r--r-- | packages/tool-web-search/src/validate.ts | 304 | ||||
| -rw-r--r-- | packages/tool-web-search/tsconfig.json | 8 |
12 files changed, 955 insertions, 955 deletions
diff --git a/packages/tool-web-search/package.json b/packages/tool-web-search/package.json index c41ab7b..3d3503f 100644 --- a/packages/tool-web-search/package.json +++ b/packages/tool-web-search/package.json @@ -1,11 +1,11 @@ { - "name": "@dispatch/tool-web-search", - "version": "0.0.0", - "type": "module", - "private": true, - "main": "dist/index.js", - "types": "dist/index.d.ts", - "dependencies": { - "@dispatch/kernel": "workspace:*" - } + "name": "@dispatch/tool-web-search", + "version": "0.0.0", + "type": "module", + "private": true, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "dependencies": { + "@dispatch/kernel": "workspace:*" + } } diff --git a/packages/tool-web-search/src/client.test.ts b/packages/tool-web-search/src/client.test.ts index f020a83..ee6e3f9 100644 --- a/packages/tool-web-search/src/client.test.ts +++ b/packages/tool-web-search/src/client.test.ts @@ -2,207 +2,207 @@ import { describe, expect, it } from "vitest"; import { createFirecrawlClient, type FetchLike } from "./client.js"; function jsonResponse(body: unknown, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: { "Content-Type": "application/json" }, - }); + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); } interface CapturedCall { - url: string; - method?: string | undefined; - body?: string | undefined; + url: string; + method?: string | undefined; + body?: string | undefined; } /** Builds a fake fetch that returns scripted responses in order, capturing each call. */ function makeFetch(responses: Response[]): { fetchFn: FetchLike; calls: CapturedCall[] } { - const calls: CapturedCall[] = []; - let i = 0; - const fetchFn: FetchLike = (async (input: string | URL | Request, init?: RequestInit) => { - const url = - typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; - calls.push({ - url, - method: init?.method, - body: typeof init?.body === "string" ? init.body : undefined, - }); - return responses[i++] ?? jsonResponse({}); - }) as unknown as FetchLike; - return { fetchFn, calls }; + const calls: CapturedCall[] = []; + let i = 0; + const fetchFn: FetchLike = (async (input: string | URL | Request, init?: RequestInit) => { + const url = + typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + calls.push({ + url, + method: init?.method, + body: typeof init?.body === "string" ? init.body : undefined, + }); + return responses[i++] ?? jsonResponse({}); + }) as unknown as FetchLike; + return { fetchFn, calls }; } const BASE = "http://test-firecrawl.local/v1"; const signal = (): AbortSignal => new AbortController().signal; describe("createFirecrawlClient.search", () => { - it("sends POST /search with correct body", async () => { - const { fetchFn, calls } = makeFetch([jsonResponse({ success: true, data: [] })]); - const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); - await client.search({ query: "hello", limit: 7 }, signal()); - - const call = calls[0]; - if (!call) throw new Error("no call captured"); - expect(call.url).toBe(`${BASE}/search`); - expect(call.method).toBe("POST"); - expect(JSON.parse(call.body ?? "{}")).toEqual({ query: "hello", limit: 7 }); - }); - - it("returns parsed data on success", async () => { - const data = [{ title: "T", url: "http://x", description: "d" }]; - const { fetchFn } = makeFetch([jsonResponse({ success: true, data })]); - const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); - const result = await client.search({ query: "hello", limit: 7 }, signal()); - expect(result).toEqual(data); - }); - - it("throws on !success", async () => { - const { fetchFn } = makeFetch([jsonResponse({ success: false, error: "boom" })]); - const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); - await expect(client.search({ query: "x", limit: 7 }, signal())).rejects.toThrow("boom"); - }); + it("sends POST /search with correct body", async () => { + const { fetchFn, calls } = makeFetch([jsonResponse({ success: true, data: [] })]); + const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); + await client.search({ query: "hello", limit: 7 }, signal()); + + const call = calls[0]; + if (!call) throw new Error("no call captured"); + expect(call.url).toBe(`${BASE}/search`); + expect(call.method).toBe("POST"); + expect(JSON.parse(call.body ?? "{}")).toEqual({ query: "hello", limit: 7 }); + }); + + it("returns parsed data on success", async () => { + const data = [{ title: "T", url: "http://x", description: "d" }]; + const { fetchFn } = makeFetch([jsonResponse({ success: true, data })]); + const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); + const result = await client.search({ query: "hello", limit: 7 }, signal()); + expect(result).toEqual(data); + }); + + it("throws on !success", async () => { + const { fetchFn } = makeFetch([jsonResponse({ success: false, error: "boom" })]); + const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); + await expect(client.search({ query: "x", limit: 7 }, signal())).rejects.toThrow("boom"); + }); }); describe("createFirecrawlClient.scrape", () => { - it("sends POST /scrape with correct body", async () => { - const { fetchFn, calls } = makeFetch([ - jsonResponse({ success: true, data: { markdown: "md", metadata: { title: "T" } } }), - ]); - const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); - await client.scrape({ url: "http://x", formats: ["markdown"] }, signal()); - - const call = calls[0]; - if (!call) throw new Error("no call captured"); - expect(call.url).toBe(`${BASE}/scrape`); - expect(call.method).toBe("POST"); - expect(JSON.parse(call.body ?? "{}")).toEqual({ - url: "http://x", - formats: ["markdown"], - onlyMainContent: true, - }); - }); + it("sends POST /scrape with correct body", async () => { + const { fetchFn, calls } = makeFetch([ + jsonResponse({ success: true, data: { markdown: "md", metadata: { title: "T" } } }), + ]); + const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); + await client.scrape({ url: "http://x", formats: ["markdown"] }, signal()); + + const call = calls[0]; + if (!call) throw new Error("no call captured"); + expect(call.url).toBe(`${BASE}/scrape`); + expect(call.method).toBe("POST"); + expect(JSON.parse(call.body ?? "{}")).toEqual({ + url: "http://x", + formats: ["markdown"], + onlyMainContent: true, + }); + }); }); describe("createFirecrawlClient.crawl", () => { - it("polls status URL until completed", async () => { - const { fetchFn, calls } = makeFetch([ - jsonResponse({ success: true, url: `${BASE}/crawl/status/123` }), - jsonResponse({ status: "scraping" }), - jsonResponse({ - status: "completed", - data: [{ markdown: "p1", metadata: { title: "P1", sourceURL: "http://p1" } }], - }), - ]); - const client = createFirecrawlClient({ - baseUrl: BASE, - fetchFn, - sleep: async () => {}, - }); - const pages = await client.crawl( - { url: "http://site", limit: 3, formats: ["markdown"] }, - signal(), - ); - expect(pages).toEqual([{ markdown: "p1", metadata: { title: "P1", sourceURL: "http://p1" } }]); - expect(calls.length).toBe(3); - }); - - it("returns data when completed", async () => { - const { fetchFn } = makeFetch([ - jsonResponse({ success: true, url: `${BASE}/crawl/status/123` }), - jsonResponse({ - status: "completed", - data: [{ markdown: "page", metadata: { title: "T" } }], - }), - ]); - const client = createFirecrawlClient({ - baseUrl: BASE, - fetchFn, - sleep: async () => {}, - }); - const pages = await client.crawl( - { url: "http://site", limit: 3, formats: ["markdown"] }, - signal(), - ); - expect(pages.length).toBe(1); - expect(pages[0]?.markdown).toBe("page"); - }); - - it("throws when status is failed", async () => { - const { fetchFn } = makeFetch([ - jsonResponse({ success: true, url: `${BASE}/crawl/status/123` }), - jsonResponse({ status: "failed", error: "boom" }), - ]); - const client = createFirecrawlClient({ - baseUrl: BASE, - fetchFn, - sleep: async () => {}, - }); - await expect( - client.crawl({ url: "http://site", limit: 3, formats: ["markdown"] }, signal()), - ).rejects.toThrow("failed"); - }); - - it("respects abort signal (stops polling)", async () => { - const controller = new AbortController(); - const { fetchFn, calls } = makeFetch([ - jsonResponse({ success: true, url: `${BASE}/crawl/status/123` }), - ]); - const client = createFirecrawlClient({ - baseUrl: BASE, - fetchFn, - sleep: async (_ms, sig) => { - controller.abort(); - if (sig.aborted) throw new Error("Request aborted."); - }, - }); - await expect( - client.crawl({ url: "http://site", limit: 3, formats: ["markdown"] }, controller.signal), - ).rejects.toThrow(); - expect(calls.length).toBe(1); - }); + it("polls status URL until completed", async () => { + const { fetchFn, calls } = makeFetch([ + jsonResponse({ success: true, url: `${BASE}/crawl/status/123` }), + jsonResponse({ status: "scraping" }), + jsonResponse({ + status: "completed", + data: [{ markdown: "p1", metadata: { title: "P1", sourceURL: "http://p1" } }], + }), + ]); + const client = createFirecrawlClient({ + baseUrl: BASE, + fetchFn, + sleep: async () => {}, + }); + const pages = await client.crawl( + { url: "http://site", limit: 3, formats: ["markdown"] }, + signal(), + ); + expect(pages).toEqual([{ markdown: "p1", metadata: { title: "P1", sourceURL: "http://p1" } }]); + expect(calls.length).toBe(3); + }); + + it("returns data when completed", async () => { + const { fetchFn } = makeFetch([ + jsonResponse({ success: true, url: `${BASE}/crawl/status/123` }), + jsonResponse({ + status: "completed", + data: [{ markdown: "page", metadata: { title: "T" } }], + }), + ]); + const client = createFirecrawlClient({ + baseUrl: BASE, + fetchFn, + sleep: async () => {}, + }); + const pages = await client.crawl( + { url: "http://site", limit: 3, formats: ["markdown"] }, + signal(), + ); + expect(pages.length).toBe(1); + expect(pages[0]?.markdown).toBe("page"); + }); + + it("throws when status is failed", async () => { + const { fetchFn } = makeFetch([ + jsonResponse({ success: true, url: `${BASE}/crawl/status/123` }), + jsonResponse({ status: "failed", error: "boom" }), + ]); + const client = createFirecrawlClient({ + baseUrl: BASE, + fetchFn, + sleep: async () => {}, + }); + await expect( + client.crawl({ url: "http://site", limit: 3, formats: ["markdown"] }, signal()), + ).rejects.toThrow("failed"); + }); + + it("respects abort signal (stops polling)", async () => { + const controller = new AbortController(); + const { fetchFn, calls } = makeFetch([ + jsonResponse({ success: true, url: `${BASE}/crawl/status/123` }), + ]); + const client = createFirecrawlClient({ + baseUrl: BASE, + fetchFn, + sleep: async (_ms, sig) => { + controller.abort(); + if (sig.aborted) throw new Error("Request aborted."); + }, + }); + await expect( + client.crawl({ url: "http://site", limit: 3, formats: ["markdown"] }, controller.signal), + ).rejects.toThrow(); + expect(calls.length).toBe(1); + }); }); describe("createFirecrawlClient.map", () => { - it("sends POST /map and returns links", async () => { - const { fetchFn, calls } = makeFetch([ - jsonResponse({ success: true, links: ["http://a", "http://b"] }), - ]); - const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); - const links = await client.map("http://site", signal()); - expect(links).toEqual(["http://a", "http://b"]); - - const call = calls[0]; - if (!call) throw new Error("no call captured"); - expect(call.url).toBe(`${BASE}/map`); - expect(call.method).toBe("POST"); - expect(JSON.parse(call.body ?? "{}")).toEqual({ url: "http://site" }); - }); + it("sends POST /map and returns links", async () => { + const { fetchFn, calls } = makeFetch([ + jsonResponse({ success: true, links: ["http://a", "http://b"] }), + ]); + const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); + const links = await client.map("http://site", signal()); + expect(links).toEqual(["http://a", "http://b"]); + + const call = calls[0]; + if (!call) throw new Error("no call captured"); + expect(call.url).toBe(`${BASE}/map`); + expect(call.method).toBe("POST"); + expect(JSON.parse(call.body ?? "{}")).toEqual({ url: "http://site" }); + }); }); describe("createFirecrawlClient.request (error paths)", () => { - it("throws on HTTP error", async () => { - const { fetchFn } = makeFetch([ - new Response("not found", { status: 404, statusText: "Not Found" }), - ]); - const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); - await expect(client.search({ query: "x", limit: 7 }, signal())).rejects.toThrow("HTTP 404"); - }); - - it("throws on timeout", async () => { - const fetchFn: FetchLike = ((_input: string | URL | Request, init?: RequestInit) => - new Promise<Response>((_resolve, reject) => { - const sig = init?.signal; - if (!sig) return; - sig.addEventListener("abort", () => { - const err = new Error("aborted"); - err.name = "AbortError"; - reject(err); - }); - })) as unknown as FetchLike; - const client = createFirecrawlClient({ - baseUrl: BASE, - fetchFn, - timeoutMs: 10, - }); - await expect(client.search({ query: "x", limit: 7 }, signal())).rejects.toThrow("timed out"); - }); + it("throws on HTTP error", async () => { + const { fetchFn } = makeFetch([ + new Response("not found", { status: 404, statusText: "Not Found" }), + ]); + const client = createFirecrawlClient({ baseUrl: BASE, fetchFn }); + await expect(client.search({ query: "x", limit: 7 }, signal())).rejects.toThrow("HTTP 404"); + }); + + it("throws on timeout", async () => { + const fetchFn: FetchLike = ((_input: string | URL | Request, init?: RequestInit) => + new Promise<Response>((_resolve, reject) => { + const sig = init?.signal; + if (!sig) return; + sig.addEventListener("abort", () => { + const err = new Error("aborted"); + err.name = "AbortError"; + reject(err); + }); + })) as unknown as FetchLike; + const client = createFirecrawlClient({ + baseUrl: BASE, + fetchFn, + timeoutMs: 10, + }); + await expect(client.search({ query: "x", limit: 7 }, signal())).rejects.toThrow("timed out"); + }); }); diff --git a/packages/tool-web-search/src/client.ts b/packages/tool-web-search/src/client.ts index 071ba97..cd137f5 100644 --- a/packages/tool-web-search/src/client.ts +++ b/packages/tool-web-search/src/client.ts @@ -17,97 +17,97 @@ export const CRAWL_POLL_MS = 2_000; export const CRAWL_MAX_WAIT_MS = 5 * 60 * 1_000; export interface SearchParams { - readonly query: string; - readonly limit: number; - readonly lang?: string; - readonly country?: string; - readonly scrapeOptions?: { - readonly formats: readonly string[]; - readonly onlyMainContent: boolean; - }; + readonly query: string; + readonly limit: number; + readonly lang?: string; + readonly country?: string; + readonly scrapeOptions?: { + readonly formats: readonly string[]; + readonly onlyMainContent: boolean; + }; } export interface ScrapeParams { - readonly url: string; - readonly formats: readonly string[]; + readonly url: string; + readonly formats: readonly string[]; } export interface CrawlParams { - readonly url: string; - readonly limit: number; - readonly formats: readonly string[]; + readonly url: string; + readonly limit: number; + readonly formats: readonly string[]; } export interface FirecrawlClient { - readonly search: (params: SearchParams, signal: AbortSignal) => Promise<readonly SearchHit[]>; - readonly scrape: (params: ScrapeParams, signal: AbortSignal) => Promise<ScrapeResult>; - readonly crawl: (params: CrawlParams, signal: AbortSignal) => Promise<readonly CrawlPage[]>; - readonly map: (url: string, signal: AbortSignal) => Promise<readonly string[]>; + readonly search: (params: SearchParams, signal: AbortSignal) => Promise<readonly SearchHit[]>; + readonly scrape: (params: ScrapeParams, signal: AbortSignal) => Promise<ScrapeResult>; + readonly crawl: (params: CrawlParams, signal: AbortSignal) => Promise<readonly CrawlPage[]>; + readonly map: (url: string, signal: AbortSignal) => Promise<readonly string[]>; } export interface FirecrawlClientDeps { - readonly baseUrl: string; - readonly fetchFn: FetchLike; - readonly timeoutMs?: number; - readonly pollMs?: number; - readonly maxWaitMs?: number; - readonly now?: () => number; - readonly sleep?: (ms: number, signal: AbortSignal) => Promise<void>; + readonly baseUrl: string; + readonly fetchFn: FetchLike; + readonly timeoutMs?: number; + readonly pollMs?: number; + readonly maxWaitMs?: number; + readonly now?: () => number; + readonly sleep?: (ms: number, signal: AbortSignal) => Promise<void>; } interface SearchResponse { - readonly success: boolean; - readonly data?: readonly SearchHit[]; - readonly error?: string; + readonly success: boolean; + readonly data?: readonly SearchHit[]; + readonly error?: string; } interface ScrapeResponse { - readonly success: boolean; - readonly data?: { - readonly markdown?: string; - readonly metadata?: { readonly title?: string }; - }; - readonly error?: string; + readonly success: boolean; + readonly data?: { + readonly markdown?: string; + readonly metadata?: { readonly title?: string }; + }; + readonly error?: string; } interface CrawlStartResponse { - readonly success: boolean; - readonly url?: string; - readonly error?: string; + readonly success: boolean; + readonly url?: string; + readonly error?: string; } interface CrawlStatusResponse { - readonly status: string; - readonly data?: readonly CrawlPage[]; - readonly error?: string; + readonly status: string; + readonly data?: readonly CrawlPage[]; + readonly error?: string; } interface MapResponse { - readonly success: boolean; - readonly links?: readonly string[]; - readonly error?: string; + readonly success: boolean; + readonly links?: readonly string[]; + readonly error?: string; } /** Default sleep: resolve after `ms`, reject on abort. */ async function defaultSleep(ms: number, signal: AbortSignal): Promise<void> { - return new Promise<void>((resolve, reject) => { - if (signal.aborted) { - reject(new Error("Request aborted.")); - return; - } - let timer: ReturnType<typeof setTimeout> | undefined; - const onAbort = (): void => { - if (timer !== undefined) { - clearTimeout(timer); - } - reject(new Error("Request aborted.")); - }; - timer = setTimeout(() => { - signal.removeEventListener("abort", onAbort); - resolve(); - }, ms); - signal.addEventListener("abort", onAbort, { once: true }); - }); + return new Promise<void>((resolve, reject) => { + if (signal.aborted) { + reject(new Error("Request aborted.")); + return; + } + let timer: ReturnType<typeof setTimeout> | undefined; + const onAbort = (): void => { + if (timer !== undefined) { + clearTimeout(timer); + } + reject(new Error("Request aborted.")); + }; + timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, ms); + signal.addEventListener("abort", onAbort, { once: true }); + }); } /** @@ -116,128 +116,128 @@ async function defaultSleep(ms: number, signal: AbortSignal): Promise<void> { * is combined with the caller's cancellation signal via `AbortSignal.any`. */ export function createFirecrawlClient(deps: FirecrawlClientDeps): FirecrawlClient { - const baseUrl = deps.baseUrl; - const fetchFn = deps.fetchFn; - const timeoutMs = deps.timeoutMs ?? DEFAULT_TIMEOUT_MS; - const pollMs = deps.pollMs ?? CRAWL_POLL_MS; - const maxWaitMs = deps.maxWaitMs ?? CRAWL_MAX_WAIT_MS; - const now = deps.now ?? Date.now; - const sleep = deps.sleep ?? defaultSleep; - - async function request( - method: "POST" | "GET", - url: string, - body: unknown, - signal: AbortSignal, - ): Promise<unknown> { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), timeoutMs); - const combined = AbortSignal.any([signal, controller.signal]); - try { - let response: Response; - try { - response = await fetchFn(url, { - method, - headers: - body !== undefined - ? { "Content-Type": "application/json", Accept: "application/json" } - : { Accept: "application/json" }, - body: body !== undefined ? JSON.stringify(body) : undefined, - signal: combined, - }); - } catch (err) { - if (signal.aborted) { - throw new Error("Request aborted."); - } - if (controller.signal.aborted) { - throw new Error(`Firecrawl request timed out after ${timeoutMs / 1000} seconds.`); - } - throw err; - } - if (!response.ok) { - const text = await response.text().catch(() => ""); - throw new Error(`HTTP ${response.status} ${response.statusText}${text ? `: ${text}` : ""}`); - } - try { - return await response.json(); - } catch { - throw new Error("Failed to parse Firecrawl response as JSON"); - } - } finally { - clearTimeout(timeout); - } - } - - async function post(endpoint: string, body: unknown, signal: AbortSignal): Promise<unknown> { - return request("POST", `${baseUrl}/${endpoint}`, body, signal); - } - - return { - async search(params: SearchParams, signal: AbortSignal): Promise<readonly SearchHit[]> { - const body: Record<string, unknown> = { query: params.query, limit: params.limit }; - if (params.lang !== undefined) { - body.lang = params.lang; - } - if (params.country !== undefined) { - body.country = params.country; - } - if (params.scrapeOptions !== undefined) { - body.scrapeOptions = params.scrapeOptions; - } - const json = (await post("search", body, signal)) as SearchResponse; - if (!json.success) { - throw new Error(json.error ?? "Unknown error"); - } - return json.data ?? []; - }, - - async scrape(params: ScrapeParams, signal: AbortSignal): Promise<ScrapeResult> { - const body = { - url: params.url, - formats: params.formats, - onlyMainContent: true, - }; - const json = (await post("scrape", body, signal)) as ScrapeResponse; - if (!json.success) { - throw new Error(json.error ?? "Unknown error"); - } - return json; - }, - - async crawl(params: CrawlParams, signal: AbortSignal): Promise<readonly CrawlPage[]> { - const body = { - url: params.url, - limit: params.limit, - scrapeOptions: { formats: params.formats, onlyMainContent: true }, - }; - const startJson = (await post("crawl", body, signal)) as CrawlStartResponse; - if (!startJson.success) { - throw new Error(startJson.error ?? "Unknown error"); - } - const statusUrl = startJson.url; - if (statusUrl === undefined) { - throw new Error("crawl response missing status URL."); - } - const started = now(); - while (now() - started < maxWaitMs) { - await sleep(pollMs, signal); - const status = (await request("GET", statusUrl, undefined, signal)) as CrawlStatusResponse; - if (status.status === "completed") { - return status.data ?? []; - } - if (status.status === "failed") { - throw new Error(`crawl failed: ${status.error ?? "unknown"}`); - } - } - throw new Error("crawl timed out waiting for completion."); - }, - - async map(url: string, signal: AbortSignal): Promise<readonly string[]> { - const json = (await post("map", { url }, signal)) as MapResponse; - if (!json.success) { - throw new Error(json.error ?? "Unknown error"); - } - return json.links ?? []; - }, - }; + const baseUrl = deps.baseUrl; + const fetchFn = deps.fetchFn; + const timeoutMs = deps.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const pollMs = deps.pollMs ?? CRAWL_POLL_MS; + const maxWaitMs = deps.maxWaitMs ?? CRAWL_MAX_WAIT_MS; + const now = deps.now ?? Date.now; + const sleep = deps.sleep ?? defaultSleep; + + async function request( + method: "POST" | "GET", + url: string, + body: unknown, + signal: AbortSignal, + ): Promise<unknown> { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const combined = AbortSignal.any([signal, controller.signal]); + try { + let response: Response; + try { + response = await fetchFn(url, { + method, + headers: + body !== undefined + ? { "Content-Type": "application/json", Accept: "application/json" } + : { Accept: "application/json" }, + body: body !== undefined ? JSON.stringify(body) : undefined, + signal: combined, + }); + } catch (err) { + if (signal.aborted) { + throw new Error("Request aborted."); + } + if (controller.signal.aborted) { + throw new Error(`Firecrawl request timed out after ${timeoutMs / 1000} seconds.`); + } + throw err; + } + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error(`HTTP ${response.status} ${response.statusText}${text ? `: ${text}` : ""}`); + } + try { + return await response.json(); + } catch { + throw new Error("Failed to parse Firecrawl response as JSON"); + } + } finally { + clearTimeout(timeout); + } + } + + async function post(endpoint: string, body: unknown, signal: AbortSignal): Promise<unknown> { + return request("POST", `${baseUrl}/${endpoint}`, body, signal); + } + + return { + async search(params: SearchParams, signal: AbortSignal): Promise<readonly SearchHit[]> { + const body: Record<string, unknown> = { query: params.query, limit: params.limit }; + if (params.lang !== undefined) { + body.lang = params.lang; + } + if (params.country !== undefined) { + body.country = params.country; + } + if (params.scrapeOptions !== undefined) { + body.scrapeOptions = params.scrapeOptions; + } + const json = (await post("search", body, signal)) as SearchResponse; + if (!json.success) { + throw new Error(json.error ?? "Unknown error"); + } + return json.data ?? []; + }, + + async scrape(params: ScrapeParams, signal: AbortSignal): Promise<ScrapeResult> { + const body = { + url: params.url, + formats: params.formats, + onlyMainContent: true, + }; + const json = (await post("scrape", body, signal)) as ScrapeResponse; + if (!json.success) { + throw new Error(json.error ?? "Unknown error"); + } + return json; + }, + + async crawl(params: CrawlParams, signal: AbortSignal): Promise<readonly CrawlPage[]> { + const body = { + url: params.url, + limit: params.limit, + scrapeOptions: { formats: params.formats, onlyMainContent: true }, + }; + const startJson = (await post("crawl", body, signal)) as CrawlStartResponse; + if (!startJson.success) { + throw new Error(startJson.error ?? "Unknown error"); + } + const statusUrl = startJson.url; + if (statusUrl === undefined) { + throw new Error("crawl response missing status URL."); + } + const started = now(); + while (now() - started < maxWaitMs) { + await sleep(pollMs, signal); + const status = (await request("GET", statusUrl, undefined, signal)) as CrawlStatusResponse; + if (status.status === "completed") { + return status.data ?? []; + } + if (status.status === "failed") { + throw new Error(`crawl failed: ${status.error ?? "unknown"}`); + } + } + throw new Error("crawl timed out waiting for completion."); + }, + + async map(url: string, signal: AbortSignal): Promise<readonly string[]> { + const json = (await post("map", { url }, signal)) as MapResponse; + if (!json.success) { + throw new Error(json.error ?? "Unknown error"); + } + return json.links ?? []; + }, + }; } diff --git a/packages/tool-web-search/src/extension.test.ts b/packages/tool-web-search/src/extension.test.ts index 6e0a6bc..f6ee198 100644 --- a/packages/tool-web-search/src/extension.test.ts +++ b/packages/tool-web-search/src/extension.test.ts @@ -3,111 +3,111 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { activate, extension, manifest } from "./extension.js"; function stubCtx(overrides?: Partial<ToolExecuteContext>): ToolExecuteContext { - return { - toolCallId: "test-call-1", - onOutput: () => {}, - signal: new AbortController().signal, - log: createLogger( - { extensionId: "test" }, - { emit: () => {} }, - { now: () => 0, newId: () => "id" }, - ), - ...overrides, - }; + return { + toolCallId: "test-call-1", + onOutput: () => {}, + signal: new AbortController().signal, + log: createLogger( + { extensionId: "test" }, + { emit: () => {} }, + { now: () => 0, newId: () => "id" }, + ), + ...overrides, + }; } function makeFakeHost(): { host: HostAPI; defineTool: ReturnType<typeof vi.fn> } { - const defineTool = vi.fn(); - const host = { - defineTool, - logger: { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - span: vi.fn(() => ({ end: vi.fn() })), - }, - } as unknown as HostAPI; - return { host, defineTool }; + const defineTool = vi.fn(); + const host = { + defineTool, + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + span: vi.fn(() => ({ end: vi.fn() })), + }, + } as unknown as HostAPI; + return { host, defineTool }; } const ORIG_FETCH = globalThis.fetch; const ORIG_ENV = process.env.FIRECRAWL_BASE_URL; function restoreEnv(): void { - if (ORIG_ENV === undefined) { - delete process.env.FIRECRAWL_BASE_URL; - } else { - process.env.FIRECRAWL_BASE_URL = ORIG_ENV; - } + if (ORIG_ENV === undefined) { + delete process.env.FIRECRAWL_BASE_URL; + } else { + process.env.FIRECRAWL_BASE_URL = ORIG_ENV; + } } afterEach(() => { - globalThis.fetch = ORIG_FETCH; - restoreEnv(); + globalThis.fetch = ORIG_FETCH; + restoreEnv(); }); function stubFetchCapture(): { calls: Array<{ url: string }> } { - const calls: Array<{ url: string }> = []; - globalThis.fetch = vi.fn(async (input: string | URL | Request) => { - calls.push({ url: String(input) }); - return new Response(JSON.stringify({ success: true, data: [] }), { - status: 200, - headers: { "Content-Type": "application/json" }, - }); - }) as unknown as typeof globalThis.fetch; - return { calls }; + const calls: Array<{ url: string }> = []; + globalThis.fetch = vi.fn(async (input: string | URL | Request) => { + calls.push({ url: String(input) }); + return new Response(JSON.stringify({ success: true, data: [] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }) as unknown as typeof globalThis.fetch; + return { calls }; } describe("tool-web-search activation", () => { - it("registers the 'web_search' tool (defineTool called)", () => { - const { host, defineTool } = makeFakeHost(); - activate(host); - expect(defineTool).toHaveBeenCalledTimes(1); - const registered = defineTool.mock.calls[0]?.[0]; - if (!registered) throw new Error("no tool registered"); - expect(registered.name).toBe("web_search"); - expect(registered.concurrencySafe).toBe(true); - }); + it("registers the 'web_search' tool (defineTool called)", () => { + const { host, defineTool } = makeFakeHost(); + activate(host); + expect(defineTool).toHaveBeenCalledTimes(1); + const registered = defineTool.mock.calls[0]?.[0]; + if (!registered) throw new Error("no tool registered"); + expect(registered.name).toBe("web_search"); + expect(registered.concurrencySafe).toBe(true); + }); - it("uses FIRECRAWL_BASE_URL from env", async () => { - process.env.FIRECRAWL_BASE_URL = "http://env-firecrawl.local/v1"; - const { calls } = stubFetchCapture(); - const { host, defineTool } = makeFakeHost(); - activate(host); + it("uses FIRECRAWL_BASE_URL from env", async () => { + process.env.FIRECRAWL_BASE_URL = "http://env-firecrawl.local/v1"; + const { calls } = stubFetchCapture(); + const { host, defineTool } = makeFakeHost(); + activate(host); - const tool = defineTool.mock.calls[0]?.[0]; - if (!tool) throw new Error("no tool registered"); - await tool.execute({ query: "hello" }, stubCtx()); - expect(calls.length).toBeGreaterThan(0); - expect(calls[0]?.url).toContain("http://env-firecrawl.local/v1/search"); - }); + const tool = defineTool.mock.calls[0]?.[0]; + if (!tool) throw new Error("no tool registered"); + await tool.execute({ query: "hello" }, stubCtx()); + expect(calls.length).toBeGreaterThan(0); + expect(calls[0]?.url).toContain("http://env-firecrawl.local/v1/search"); + }); - it("uses default base URL when env unset", async () => { - delete process.env.FIRECRAWL_BASE_URL; - const { calls } = stubFetchCapture(); - const { host, defineTool } = makeFakeHost(); - activate(host); + it("uses default base URL when env unset", async () => { + delete process.env.FIRECRAWL_BASE_URL; + const { calls } = stubFetchCapture(); + const { host, defineTool } = makeFakeHost(); + activate(host); - const tool = defineTool.mock.calls[0]?.[0]; - if (!tool) throw new Error("no tool registered"); - await tool.execute({ query: "hello" }, stubCtx()); - expect(calls.length).toBeGreaterThan(0); - expect(calls[0]?.url).toContain("100.102.55.49:31329/v1/search"); - }); + const tool = defineTool.mock.calls[0]?.[0]; + if (!tool) throw new Error("no tool registered"); + await tool.execute({ query: "hello" }, stubCtx()); + expect(calls.length).toBeGreaterThan(0); + expect(calls[0]?.url).toContain("100.102.55.49:31329/v1/search"); + }); }); describe("tool-web-search manifest", () => { - it("declares network capability + web_search contribution", () => { - expect(manifest.id).toBe("tool-web-search"); - expect(manifest.capabilities).toEqual({ network: true }); - expect(manifest.contributes).toEqual({ tools: ["web_search"] }); - expect(manifest.trust).toBe("bundled"); - expect(manifest.activation).toBe("eager"); - }); + it("declares network capability + web_search contribution", () => { + expect(manifest.id).toBe("tool-web-search"); + expect(manifest.capabilities).toEqual({ network: true }); + expect(manifest.contributes).toEqual({ tools: ["web_search"] }); + expect(manifest.trust).toBe("bundled"); + expect(manifest.activation).toBe("eager"); + }); - it("extension bundles the manifest + activate", () => { - expect(extension.manifest).toBe(manifest); - expect(typeof extension.activate).toBe("function"); - }); + it("extension bundles the manifest + activate", () => { + expect(extension.manifest).toBe(manifest); + expect(typeof extension.activate).toBe("function"); + }); }); diff --git a/packages/tool-web-search/src/extension.ts b/packages/tool-web-search/src/extension.ts index 1d1803d..7eac643 100644 --- a/packages/tool-web-search/src/extension.ts +++ b/packages/tool-web-search/src/extension.ts @@ -13,20 +13,20 @@ import { createFirecrawlClient, DEFAULT_BASE_URL } from "./client.js"; import { createWebSearchTool } from "./tool.js"; export const manifest: Manifest = { - id: "tool-web-search", - name: "Web Search Tool", - version: "0.0.0", - apiVersion: "^0.1.0", - trust: "bundled", - activation: "eager", - capabilities: { network: true }, - contributes: { tools: ["web_search"] }, + id: "tool-web-search", + name: "Web Search Tool", + version: "0.0.0", + apiVersion: "^0.1.0", + trust: "bundled", + activation: "eager", + capabilities: { network: true }, + contributes: { tools: ["web_search"] }, }; export function activate(host: HostAPI): void { - const baseUrl = process.env.FIRECRAWL_BASE_URL ?? DEFAULT_BASE_URL; - const client = createFirecrawlClient({ baseUrl, fetchFn: globalThis.fetch }); - host.defineTool(createWebSearchTool({ client })); + const baseUrl = process.env.FIRECRAWL_BASE_URL ?? DEFAULT_BASE_URL; + const client = createFirecrawlClient({ baseUrl, fetchFn: globalThis.fetch }); + host.defineTool(createWebSearchTool({ client })); } export const extension: Extension = { manifest, activate }; diff --git a/packages/tool-web-search/src/format.test.ts b/packages/tool-web-search/src/format.test.ts index b98bc02..a3f8c09 100644 --- a/packages/tool-web-search/src/format.test.ts +++ b/packages/tool-web-search/src/format.test.ts @@ -1,87 +1,87 @@ import { describe, expect, it } from "vitest"; import { - formatCrawlResults, - formatMapResults, - formatScrapeResult, - formatSearchResults, - truncateOutput, + formatCrawlResults, + formatMapResults, + formatScrapeResult, + formatSearchResults, + truncateOutput, } from "./format.js"; describe("formatSearchResults", () => { - it("formats title + url + description + optional markdown", () => { - const out = formatSearchResults([ - { title: "T1", url: "http://a", description: "desc", markdown: "md-body" }, - ]); - expect(out).toBe("### T1\nhttp://a\n\ndesc\n\nmd-body"); - }); + it("formats title + url + description + optional markdown", () => { + const out = formatSearchResults([ + { title: "T1", url: "http://a", description: "desc", markdown: "md-body" }, + ]); + expect(out).toBe("### T1\nhttp://a\n\ndesc\n\nmd-body"); + }); - it("joins multiple results with ---", () => { - const out = formatSearchResults([ - { title: "T1", url: "http://a", description: "d1" }, - { title: "T2", url: "http://b", description: "d2" }, - ]); - expect(out).toBe("### T1\nhttp://a\n\nd1\n\n---\n\n### T2\nhttp://b\n\nd2"); - }); + it("joins multiple results with ---", () => { + const out = formatSearchResults([ + { title: "T1", url: "http://a", description: "d1" }, + { title: "T2", url: "http://b", description: "d2" }, + ]); + expect(out).toBe("### T1\nhttp://a\n\nd1\n\n---\n\n### T2\nhttp://b\n\nd2"); + }); - it("empty data returns 'No results found.'", () => { - expect(formatSearchResults([])).toBe("No results found."); - expect(formatSearchResults(null)).toBe("No results found."); - expect(formatSearchResults(undefined)).toBe("No results found."); - }); + it("empty data returns 'No results found.'", () => { + expect(formatSearchResults([])).toBe("No results found."); + expect(formatSearchResults(null)).toBe("No results found."); + expect(formatSearchResults(undefined)).toBe("No results found."); + }); }); describe("formatScrapeResult", () => { - it("formats title + markdown", () => { - const out = formatScrapeResult({ - data: { markdown: "body", metadata: { title: "Title" } }, - }); - expect(out).toBe("# Title\n\nbody"); - }); + it("formats title + markdown", () => { + const out = formatScrapeResult({ + data: { markdown: "body", metadata: { title: "Title" } }, + }); + expect(out).toBe("# Title\n\nbody"); + }); - it("omits title header when absent", () => { - const out = formatScrapeResult({ data: { markdown: "body" } }); - expect(out).toBe("body"); - }); + it("omits title header when absent", () => { + const out = formatScrapeResult({ data: { markdown: "body" } }); + expect(out).toBe("body"); + }); }); describe("formatCrawlResults", () => { - it("formats multiple pages", () => { - const out = formatCrawlResults([ - { markdown: "p1", metadata: { title: "P1", sourceURL: "http://p1" } }, - { markdown: "p2", metadata: { title: "P2", url: "http://p2" } }, - ]); - expect(out).toBe("## P1\nhttp://p1\n\np1\n\n---\n\n## P2\nhttp://p2\n\np2"); - }); + it("formats multiple pages", () => { + const out = formatCrawlResults([ + { markdown: "p1", metadata: { title: "P1", sourceURL: "http://p1" } }, + { markdown: "p2", metadata: { title: "P2", url: "http://p2" } }, + ]); + expect(out).toBe("## P1\nhttp://p1\n\np1\n\n---\n\n## P2\nhttp://p2\n\np2"); + }); - it("empty data returns 'No pages crawled.'", () => { - expect(formatCrawlResults([])).toBe("No pages crawled."); - expect(formatCrawlResults(null)).toBe("No pages crawled."); - }); + it("empty data returns 'No pages crawled.'", () => { + expect(formatCrawlResults([])).toBe("No pages crawled."); + expect(formatCrawlResults(null)).toBe("No pages crawled."); + }); }); describe("formatMapResults", () => { - it("formats links as bullet list", () => { - const out = formatMapResults(["http://a", "http://b"]); - expect(out).toBe("- http://a\n- http://b"); - }); + it("formats links as bullet list", () => { + const out = formatMapResults(["http://a", "http://b"]); + expect(out).toBe("- http://a\n- http://b"); + }); - it("empty links returns 'No links found.'", () => { - expect(formatMapResults([])).toBe("No links found."); - expect(formatMapResults(null)).toBe("No links found."); - }); + it("empty links returns 'No links found.'", () => { + expect(formatMapResults([])).toBe("No links found."); + expect(formatMapResults(null)).toBe("No links found."); + }); }); describe("truncateOutput", () => { - it("truncates with notice when over cap", () => { - const output = "a".repeat(100); - const result = truncateOutput(output, 50); - expect(result).toContain("a".repeat(50)); - expect(result).toContain("[Output truncated: exceeded 50 characters]"); - expect(result.length).toBeLessThan(output.length + 100); - }); + it("truncates with notice when over cap", () => { + const output = "a".repeat(100); + const result = truncateOutput(output, 50); + expect(result).toContain("a".repeat(50)); + expect(result).toContain("[Output truncated: exceeded 50 characters]"); + expect(result.length).toBeLessThan(output.length + 100); + }); - it("returns as-is when under cap", () => { - expect(truncateOutput("short", 100)).toBe("short"); - expect(truncateOutput("exact", 5)).toBe("exact"); - }); + it("returns as-is when under cap", () => { + expect(truncateOutput("short", 100)).toBe("short"); + expect(truncateOutput("exact", 5)).toBe("exact"); + }); }); diff --git a/packages/tool-web-search/src/format.ts b/packages/tool-web-search/src/format.ts index cfc9aa0..172fd85 100644 --- a/packages/tool-web-search/src/format.ts +++ b/packages/tool-web-search/src/format.ts @@ -8,28 +8,28 @@ /** A single search hit from Firecrawl's `/search` endpoint. */ export interface SearchHit { - readonly title?: string; - readonly url?: string; - readonly description?: string; - readonly markdown?: string; + readonly title?: string; + readonly url?: string; + readonly description?: string; + readonly markdown?: string; } /** One page from a completed crawl (`/crawl` status `data`). */ export interface CrawlPage { - readonly markdown?: string; - readonly metadata?: { - readonly title?: string; - readonly sourceURL?: string; - readonly url?: string; - }; + readonly markdown?: string; + readonly metadata?: { + readonly title?: string; + readonly sourceURL?: string; + readonly url?: string; + }; } /** The scrape response payload (`/scrape` `data`). */ export interface ScrapeResult { - readonly data?: { - readonly markdown?: string; - readonly metadata?: { readonly title?: string }; - }; + readonly data?: { + readonly markdown?: string; + readonly metadata?: { readonly title?: string }; + }; } /** @@ -37,11 +37,11 @@ export interface ScrapeResult { * spirit to tool-shell. Duplication across features is the intended trade. */ export function truncateOutput(output: string, cap: number): string { - if (output.length <= cap) { - return output; - } - const truncated = output.slice(0, cap); - return `${truncated}\n\n[Output truncated: exceeded ${cap} characters]`; + if (output.length <= cap) { + return output; + } + const truncated = output.slice(0, cap); + return `${truncated}\n\n[Output truncated: exceeded ${cap} characters]`; } /** @@ -49,21 +49,21 @@ export function truncateOutput(output: string, cap: number): string { * joined by `---` separators. Empty → `"No results found."`. */ export function formatSearchResults(data: readonly SearchHit[] | null | undefined): string { - if (!data || data.length === 0) { - return "No results found."; - } - const parts: string[] = []; - for (const r of data) { - const title = r.title ?? "(no title)"; - const url = r.url ?? ""; - const description = r.description ?? ""; - let section = `### ${title}\n${url}\n\n${description}`; - if (r.markdown) { - section += `\n\n${r.markdown}`; - } - parts.push(section); - } - return parts.join("\n\n---\n\n"); + if (!data || data.length === 0) { + return "No results found."; + } + const parts: string[] = []; + for (const r of data) { + const title = r.title ?? "(no title)"; + const url = r.url ?? ""; + const description = r.description ?? ""; + let section = `### ${title}\n${url}\n\n${description}`; + if (r.markdown) { + section += `\n\n${r.markdown}`; + } + parts.push(section); + } + return parts.join("\n\n---\n\n"); } /** @@ -71,12 +71,12 @@ export function formatSearchResults(data: readonly SearchHit[] | null | undefine * the title is absent. */ export function formatScrapeResult(json: ScrapeResult): string { - const md = json.data?.markdown ?? ""; - const title = json.data?.metadata?.title; - if (title) { - return `# ${title}\n\n${md}`; - } - return md; + const md = json.data?.markdown ?? ""; + const title = json.data?.metadata?.title; + if (title) { + return `# ${title}\n\n${md}`; + } + return md; } /** @@ -84,28 +84,28 @@ export function formatScrapeResult(json: ScrapeResult): string { * Empty → `"No pages crawled."`. */ export function formatCrawlResults(data: readonly CrawlPage[] | null | undefined): string { - if (!data || data.length === 0) { - return "No pages crawled."; - } - const parts: string[] = []; - for (const page of data) { - const title = page.metadata?.title ?? "(no title)"; - const url = page.metadata?.sourceURL ?? page.metadata?.url ?? ""; - let section = `## ${title}\n${url}`; - if (page.markdown) { - section += `\n\n${page.markdown}`; - } - parts.push(section); - } - return parts.join("\n\n---\n\n"); + if (!data || data.length === 0) { + return "No pages crawled."; + } + const parts: string[] = []; + for (const page of data) { + const title = page.metadata?.title ?? "(no title)"; + const url = page.metadata?.sourceURL ?? page.metadata?.url ?? ""; + let section = `## ${title}\n${url}`; + if (page.markdown) { + section += `\n\n${page.markdown}`; + } + parts.push(section); + } + return parts.join("\n\n---\n\n"); } /** * Format discovered links as a bullet list. Empty → `"No links found."`. */ export function formatMapResults(links: readonly string[] | null | undefined): string { - if (!links || links.length === 0) { - return "No links found."; - } - return links.map((l) => `- ${l}`).join("\n"); + if (!links || links.length === 0) { + return "No links found."; + } + return links.map((l) => `- ${l}`).join("\n"); } diff --git a/packages/tool-web-search/src/index.ts b/packages/tool-web-search/src/index.ts index 69894d1..519a1fe 100644 --- a/packages/tool-web-search/src/index.ts +++ b/packages/tool-web-search/src/index.ts @@ -1,40 +1,40 @@ export { - CRAWL_MAX_WAIT_MS, - CRAWL_POLL_MS, - type CrawlParams, - createFirecrawlClient, - DEFAULT_BASE_URL, - DEFAULT_TIMEOUT_MS, - type FetchLike, - type FirecrawlClient, - type FirecrawlClientDeps, - type ScrapeParams, - type SearchParams, + CRAWL_MAX_WAIT_MS, + CRAWL_POLL_MS, + type CrawlParams, + createFirecrawlClient, + DEFAULT_BASE_URL, + DEFAULT_TIMEOUT_MS, + type FetchLike, + type FirecrawlClient, + type FirecrawlClientDeps, + type ScrapeParams, + type SearchParams, } from "./client.js"; export { activate, extension, manifest } from "./extension.js"; export { - type CrawlPage, - formatCrawlResults, - formatMapResults, - formatScrapeResult, - formatSearchResults, - type ScrapeResult, - type SearchHit, - truncateOutput, + type CrawlPage, + formatCrawlResults, + formatMapResults, + formatScrapeResult, + formatSearchResults, + type ScrapeResult, + type SearchHit, + truncateOutput, } from "./format.js"; export { createWebSearchTool, type WebSearchToolDeps } from "./tool.js"; export { - CRAWL_DEFAULT_LIMIT, - type CrawlArgs, - FORMATS, - type Format, - MAX_LIMIT, - type MapArgs, - MODES, - type Mode, - type ScrapeArgs, - SEARCH_DEFAULT_LIMIT, - type SearchArgs, - type ValidatedArgs, - validateArgs, + CRAWL_DEFAULT_LIMIT, + type CrawlArgs, + FORMATS, + type Format, + MAX_LIMIT, + type MapArgs, + MODES, + type Mode, + type ScrapeArgs, + SEARCH_DEFAULT_LIMIT, + type SearchArgs, + type ValidatedArgs, + validateArgs, } from "./validate.js"; diff --git a/packages/tool-web-search/src/tool.ts b/packages/tool-web-search/src/tool.ts index 751278d..b2e84fe 100644 --- a/packages/tool-web-search/src/tool.ts +++ b/packages/tool-web-search/src/tool.ts @@ -10,11 +10,11 @@ import type { ToolContract, ToolExecuteContext, ToolResult } from "@dispatch/kernel"; import type { FirecrawlClient } from "./client.js"; import { - formatCrawlResults, - formatMapResults, - formatScrapeResult, - formatSearchResults, - truncateOutput, + formatCrawlResults, + formatMapResults, + formatScrapeResult, + formatSearchResults, + truncateOutput, } from "./format.js"; import type { ValidatedArgs } from "./validate.js"; import { validateArgs } from "./validate.js"; @@ -22,51 +22,51 @@ import { validateArgs } from "./validate.js"; const OUTPUT_CAP = 50_000; export interface WebSearchToolDeps { - readonly client: FirecrawlClient; - readonly outputCap?: number; + readonly client: FirecrawlClient; + readonly outputCap?: number; } /** Dispatch validated args to the right client method and format the result. */ async function runMode( - validated: ValidatedArgs, - client: FirecrawlClient, - signal: AbortSignal, + validated: ValidatedArgs, + client: FirecrawlClient, + signal: AbortSignal, ): Promise<string> { - switch (validated.mode) { - case "search": { - const hits = await client.search( - { - query: validated.query, - limit: validated.limit, - ...(validated.scrape - ? { scrapeOptions: { formats: ["markdown"], onlyMainContent: true } } - : {}), - ...(validated.lang !== undefined ? { lang: validated.lang } : {}), - ...(validated.country !== undefined ? { country: validated.country } : {}), - }, - signal, - ); - return formatSearchResults(hits); - } - case "scrape": { - const result = await client.scrape( - { url: validated.url, formats: [validated.format] }, - signal, - ); - return formatScrapeResult(result); - } - case "crawl": { - const pages = await client.crawl( - { url: validated.url, limit: validated.limit, formats: [validated.format] }, - signal, - ); - return formatCrawlResults(pages); - } - case "map": { - const links = await client.map(validated.url, signal); - return formatMapResults(links); - } - } + switch (validated.mode) { + case "search": { + const hits = await client.search( + { + query: validated.query, + limit: validated.limit, + ...(validated.scrape + ? { scrapeOptions: { formats: ["markdown"], onlyMainContent: true } } + : {}), + ...(validated.lang !== undefined ? { lang: validated.lang } : {}), + ...(validated.country !== undefined ? { country: validated.country } : {}), + }, + signal, + ); + return formatSearchResults(hits); + } + case "scrape": { + const result = await client.scrape( + { url: validated.url, formats: [validated.format] }, + signal, + ); + return formatScrapeResult(result); + } + case "crawl": { + const pages = await client.crawl( + { url: validated.url, limit: validated.limit, formats: [validated.format] }, + signal, + ); + return formatCrawlResults(pages); + } + case "map": { + const links = await client.map(validated.url, signal); + return formatMapResults(links); + } + } } /** @@ -75,68 +75,68 @@ async function runMode( * is declared on the extension manifest (not the tool contract). */ export function createWebSearchTool(deps: WebSearchToolDeps): ToolContract { - const client = deps.client; - const cap = deps.outputCap ?? OUTPUT_CAP; + const client = deps.client; + const cap = deps.outputCap ?? OUTPUT_CAP; - return { - name: "web_search", - description: - "Access the web via a self-hosted Firecrawl instance. Supports search, " + - "single-page scrape, site crawling, and sitemap discovery.", - parameters: { - type: "object", - properties: { - query: { type: "string", description: "The search query (search mode)." }, - url: { type: "string", description: "A URL to scrape, crawl, or map." }, - mode: { - type: "string", - enum: ["search", "scrape", "crawl", "map"], - description: - "Operation mode. 'search' (default when query present), 'scrape' " + - "(default when url present), 'crawl' (recursively scrape pages from a site), " + - "'map' (discover URLs on a site).", - }, - limit: { - type: "number", - description: "Max results. Search: default 7, max 10. Crawl: default 3, max 10.", - }, - scrape: { - type: "boolean", - description: "When searching, also scrape full markdown content of each result page.", - }, - lang: { - type: "string", - description: 'Language code to filter search results (e.g. "en", "ja").', - }, - country: { - type: "string", - description: 'Country code to filter search results (e.g. "us", "jp").', - }, - format: { - type: "string", - enum: ["markdown", "text", "html"], - description: "Format for scrape/crawl output (default: markdown).", - }, - }, - }, - concurrencySafe: true, - async execute(args: unknown, ctx: ToolExecuteContext): Promise<ToolResult> { - const validated = validateArgs(args); - if ("error" in validated) { - return { content: validated.error, isError: true }; - } - const span = ctx.log.span("web_search.execute", { mode: validated.mode }); - try { - const output = await runMode(validated, client, ctx.signal); - span.end(); - return { content: truncateOutput(output, cap) }; - } catch (err: unknown) { - span.end({ err }); - return { - content: `Error: ${err instanceof Error ? err.message : String(err)}`, - isError: true, - }; - } - }, - }; + return { + name: "web_search", + description: + "Access the web via a self-hosted Firecrawl instance. Supports search, " + + "single-page scrape, site crawling, and sitemap discovery.", + parameters: { + type: "object", + properties: { + query: { type: "string", description: "The search query (search mode)." }, + url: { type: "string", description: "A URL to scrape, crawl, or map." }, + mode: { + type: "string", + enum: ["search", "scrape", "crawl", "map"], + description: + "Operation mode. 'search' (default when query present), 'scrape' " + + "(default when url present), 'crawl' (recursively scrape pages from a site), " + + "'map' (discover URLs on a site).", + }, + limit: { + type: "number", + description: "Max results. Search: default 7, max 10. Crawl: default 3, max 10.", + }, + scrape: { + type: "boolean", + description: "When searching, also scrape full markdown content of each result page.", + }, + lang: { + type: "string", + description: 'Language code to filter search results (e.g. "en", "ja").', + }, + country: { + type: "string", + description: 'Country code to filter search results (e.g. "us", "jp").', + }, + format: { + type: "string", + enum: ["markdown", "text", "html"], + description: "Format for scrape/crawl output (default: markdown).", + }, + }, + }, + concurrencySafe: true, + async execute(args: unknown, ctx: ToolExecuteContext): Promise<ToolResult> { + const validated = validateArgs(args); + if ("error" in validated) { + return { content: validated.error, isError: true }; + } + const span = ctx.log.span("web_search.execute", { mode: validated.mode }); + try { + const output = await runMode(validated, client, ctx.signal); + span.end(); + return { content: truncateOutput(output, cap) }; + } catch (err: unknown) { + span.end({ err }); + return { + content: `Error: ${err instanceof Error ? err.message : String(err)}`, + isError: true, + }; + } + }, + }; } diff --git a/packages/tool-web-search/src/validate.test.ts b/packages/tool-web-search/src/validate.test.ts index 30ae26c..ca1aa7f 100644 --- a/packages/tool-web-search/src/validate.test.ts +++ b/packages/tool-web-search/src/validate.test.ts @@ -1,92 +1,92 @@ import { describe, expect, it } from "vitest"; import { - type CrawlArgs, - type MapArgs, - type ScrapeArgs, - type SearchArgs, - validateArgs, + type CrawlArgs, + type MapArgs, + type ScrapeArgs, + type SearchArgs, + validateArgs, } from "./validate.js"; describe("validateArgs", () => { - it("mode defaults to search when query present", () => { - const result = validateArgs({ query: "hello" }); - expect("error" in result).toBe(false); - if ("error" in result) return; - expect(result.mode).toBe("search"); - expect((result as SearchArgs).query).toBe("hello"); - }); + it("mode defaults to search when query present", () => { + const result = validateArgs({ query: "hello" }); + expect("error" in result).toBe(false); + if ("error" in result) return; + expect(result.mode).toBe("search"); + expect((result as SearchArgs).query).toBe("hello"); + }); - it("mode defaults to scrape when url present (no query)", () => { - const result = validateArgs({ url: "http://example.com" }); - expect("error" in result).toBe(false); - if ("error" in result) return; - expect(result.mode).toBe("scrape"); - expect((result as ScrapeArgs).url).toBe("http://example.com"); - }); + it("mode defaults to scrape when url present (no query)", () => { + const result = validateArgs({ url: "http://example.com" }); + expect("error" in result).toBe(false); + if ("error" in result) return; + expect(result.mode).toBe("scrape"); + expect((result as ScrapeArgs).url).toBe("http://example.com"); + }); - it("explicit mode overrides defaults", () => { - const result = validateArgs({ query: "hello", url: "http://x", mode: "map" }); - expect("error" in result).toBe(false); - if ("error" in result) return; - expect(result.mode).toBe("map"); - expect((result as MapArgs).url).toBe("http://x"); - }); + it("explicit mode overrides defaults", () => { + const result = validateArgs({ query: "hello", url: "http://x", mode: "map" }); + expect("error" in result).toBe(false); + if ("error" in result) return; + expect(result.mode).toBe("map"); + expect((result as MapArgs).url).toBe("http://x"); + }); - it("search mode requires query", () => { - const result = validateArgs({ mode: "search" }); - expect(result).toHaveProperty("error"); - }); + it("search mode requires query", () => { + const result = validateArgs({ mode: "search" }); + expect(result).toHaveProperty("error"); + }); - it("scrape/crawl/map modes require url", () => { - expect(validateArgs({ mode: "scrape" })).toHaveProperty("error"); - expect(validateArgs({ mode: "crawl" })).toHaveProperty("error"); - expect(validateArgs({ mode: "map" })).toHaveProperty("error"); - }); + it("scrape/crawl/map modes require url", () => { + expect(validateArgs({ mode: "scrape" })).toHaveProperty("error"); + expect(validateArgs({ mode: "crawl" })).toHaveProperty("error"); + expect(validateArgs({ mode: "map" })).toHaveProperty("error"); + }); - it("limit clamped to max 10", () => { - const result = validateArgs({ query: "hello", limit: 50 }); - expect("error" in result).toBe(false); - if ("error" in result) return; - expect((result as SearchArgs).limit).toBe(10); - }); + it("limit clamped to max 10", () => { + const result = validateArgs({ query: "hello", limit: 50 }); + expect("error" in result).toBe(false); + if ("error" in result) return; + expect((result as SearchArgs).limit).toBe(10); + }); - it("limit defaults to 7 (search) / 3 (crawl)", () => { - const search = validateArgs({ query: "hello" }); - expect("error" in search).toBe(false); - if ("error" in search) return; - expect((search as SearchArgs).limit).toBe(7); + it("limit defaults to 7 (search) / 3 (crawl)", () => { + const search = validateArgs({ query: "hello" }); + expect("error" in search).toBe(false); + if ("error" in search) return; + expect((search as SearchArgs).limit).toBe(7); - const crawl = validateArgs({ url: "http://x", mode: "crawl" }); - expect("error" in crawl).toBe(false); - if ("error" in crawl) return; - expect((crawl as CrawlArgs).limit).toBe(3); - }); + const crawl = validateArgs({ url: "http://x", mode: "crawl" }); + expect("error" in crawl).toBe(false); + if ("error" in crawl) return; + expect((crawl as CrawlArgs).limit).toBe(3); + }); - it("format defaults to markdown", () => { - const result = validateArgs({ query: "hello" }); - expect("error" in result).toBe(false); - if ("error" in result) return; - expect(result.format).toBe("markdown"); - }); + it("format defaults to markdown", () => { + const result = validateArgs({ query: "hello" }); + expect("error" in result).toBe(false); + if ("error" in result) return; + expect(result.format).toBe("markdown"); + }); - it("rejects invalid mode", () => { - const result = validateArgs({ mode: "invalid" }); - expect(result).toHaveProperty("error"); - if (!("error" in result)) return; - expect(result.error).toContain("Invalid mode"); - }); + it("rejects invalid mode", () => { + const result = validateArgs({ mode: "invalid" }); + expect(result).toHaveProperty("error"); + if (!("error" in result)) return; + expect(result.error).toContain("Invalid mode"); + }); - it("rejects invalid format", () => { - const result = validateArgs({ url: "http://x", format: "pdf" }); - expect(result).toHaveProperty("error"); - if (!("error" in result)) return; - expect(result.error).toContain("Invalid format"); - }); + it("rejects invalid format", () => { + const result = validateArgs({ url: "http://x", format: "pdf" }); + expect(result).toHaveProperty("error"); + if (!("error" in result)) return; + expect(result.error).toContain("Invalid format"); + }); - it("returns error for null/non-object args", () => { - expect(validateArgs(null)).toHaveProperty("error"); - expect(validateArgs(undefined)).toHaveProperty("error"); - expect(validateArgs("string")).toHaveProperty("error"); - expect(validateArgs(42)).toHaveProperty("error"); - }); + it("returns error for null/non-object args", () => { + expect(validateArgs(null)).toHaveProperty("error"); + expect(validateArgs(undefined)).toHaveProperty("error"); + expect(validateArgs("string")).toHaveProperty("error"); + expect(validateArgs(42)).toHaveProperty("error"); + }); }); diff --git a/packages/tool-web-search/src/validate.ts b/packages/tool-web-search/src/validate.ts index 56bd356..019fd5a 100644 --- a/packages/tool-web-search/src/validate.ts +++ b/packages/tool-web-search/src/validate.ts @@ -17,32 +17,32 @@ export const CRAWL_DEFAULT_LIMIT = 3; export const MAX_LIMIT = 10; interface BaseArgs { - readonly format: Format; + readonly format: Format; } export interface SearchArgs extends BaseArgs { - readonly mode: "search"; - readonly query: string; - readonly limit: number; - readonly scrape: boolean; - readonly lang?: string; - readonly country?: string; + readonly mode: "search"; + readonly query: string; + readonly limit: number; + readonly scrape: boolean; + readonly lang?: string; + readonly country?: string; } export interface ScrapeArgs extends BaseArgs { - readonly mode: "scrape"; - readonly url: string; + readonly mode: "scrape"; + readonly url: string; } export interface CrawlArgs extends BaseArgs { - readonly mode: "crawl"; - readonly url: string; - readonly limit: number; + readonly mode: "crawl"; + readonly url: string; + readonly limit: number; } export interface MapArgs extends BaseArgs { - readonly mode: "map"; - readonly url: string; + readonly mode: "map"; + readonly url: string; } export type ValidatedArgs = SearchArgs | ScrapeArgs | CrawlArgs | MapArgs; @@ -52,60 +52,60 @@ export type ValidationError = { readonly error: string }; type Result<T> = { readonly value: T } | ValidationError; function resolveFormat(raw: unknown): Result<Format> { - if (raw === undefined || raw === null) { - return { value: "markdown" }; - } - if (typeof raw === "string" && (FORMATS as readonly string[]).includes(raw)) { - return { value: raw as Format }; - } - return { - error: `Error: Invalid format "${String(raw)}" (must be one of: markdown, text, html).`, - }; + if (raw === undefined || raw === null) { + return { value: "markdown" }; + } + if (typeof raw === "string" && (FORMATS as readonly string[]).includes(raw)) { + return { value: raw as Format }; + } + return { + error: `Error: Invalid format "${String(raw)}" (must be one of: markdown, text, html).`, + }; } function resolveMode(raw: unknown, query: unknown, url: unknown): Result<Mode> { - if (raw === undefined || raw === null) { - const hasQuery = typeof query === "string" && query.trim().length > 0; - const hasUrl = typeof url === "string" && url.trim().length > 0; - return { value: hasQuery ? "search" : hasUrl ? "scrape" : "search" }; - } - if (typeof raw === "string" && (MODES as readonly string[]).includes(raw)) { - return { value: raw as Mode }; - } - return { - error: `Error: Invalid mode "${String(raw)}" (must be one of: search, scrape, crawl, map).`, - }; + if (raw === undefined || raw === null) { + const hasQuery = typeof query === "string" && query.trim().length > 0; + const hasUrl = typeof url === "string" && url.trim().length > 0; + return { value: hasQuery ? "search" : hasUrl ? "scrape" : "search" }; + } + if (typeof raw === "string" && (MODES as readonly string[]).includes(raw)) { + return { value: raw as Mode }; + } + return { + error: `Error: Invalid mode "${String(raw)}" (must be one of: search, scrape, crawl, map).`, + }; } function optionalString(raw: unknown, name: string): Result<string | undefined> { - if (raw === undefined || raw === null) { - return { value: undefined }; - } - if (typeof raw === "string") { - return { value: raw }; - } - return { error: `Error: "${name}" must be a string.` }; + if (raw === undefined || raw === null) { + return { value: undefined }; + } + if (typeof raw === "string") { + return { value: raw }; + } + return { error: `Error: "${name}" must be a string.` }; } function resolveLimit(raw: unknown, defaultLimit: number): Result<number> { - if (raw === undefined || raw === null) { - return { value: defaultLimit }; - } - const n = Number(raw); - if (!Number.isFinite(n) || n < 1) { - return { error: 'Error: "limit" must be a positive number.' }; - } - return { value: Math.min(Math.floor(n), MAX_LIMIT) }; + if (raw === undefined || raw === null) { + return { value: defaultLimit }; + } + const n = Number(raw); + if (!Number.isFinite(n) || n < 1) { + return { error: 'Error: "limit" must be a positive number.' }; + } + return { value: Math.min(Math.floor(n), MAX_LIMIT) }; } function resolveBoolean(raw: unknown, name: string): Result<boolean> { - if (raw === undefined || raw === null) { - return { value: false }; - } - if (typeof raw === "boolean") { - return { value: raw }; - } - return { error: `Error: "${name}" must be a boolean.` }; + if (raw === undefined || raw === null) { + return { value: false }; + } + if (typeof raw === "boolean") { + return { value: raw }; + } + return { error: `Error: "${name}" must be a boolean.` }; } /** @@ -113,100 +113,100 @@ function resolveBoolean(raw: unknown, name: string): Result<boolean> { * Returns `{ error }` for invalid input — the tool surfaces it verbatim. */ export function validateArgs(args: unknown): ValidatedArgs | ValidationError { - if (args === null || args === undefined || typeof args !== "object") { - return { error: "Error: Arguments must be an object." }; - } - const obj = args as Record<string, unknown>; - - const format = resolveFormat(obj.format); - if ("error" in format) { - return format; - } - - const mode = resolveMode(obj.mode, obj.query, obj.url); - if ("error" in mode) { - return mode; - } - - const query = optionalString(obj.query, "query"); - if ("error" in query) { - return query; - } - - const url = optionalString(obj.url, "url"); - if ("error" in url) { - return url; - } - - switch (mode.value) { - case "search": { - if (query.value === undefined || query.value.trim().length === 0) { - return { error: "Error: query is required for search mode." }; - } - const limit = resolveLimit(obj.limit, SEARCH_DEFAULT_LIMIT); - if ("error" in limit) { - return limit; - } - const scrape = resolveBoolean(obj.scrape, "scrape"); - if ("error" in scrape) { - return scrape; - } - const lang = optionalString(obj.lang, "lang"); - if ("error" in lang) { - return lang; - } - const country = optionalString(obj.country, "country"); - if ("error" in country) { - return country; - } - const result: SearchArgs = { - mode: "search", - query: query.value, - limit: limit.value, - scrape: scrape.value, - format: format.value, - ...(lang.value !== undefined ? { lang: lang.value } : {}), - ...(country.value !== undefined ? { country: country.value } : {}), - }; - return result; - } - case "scrape": { - if (url.value === undefined || url.value.trim().length === 0) { - return { error: "Error: url is required for scrape mode." }; - } - const result: ScrapeArgs = { - mode: "scrape", - url: url.value, - format: format.value, - }; - return result; - } - case "crawl": { - if (url.value === undefined || url.value.trim().length === 0) { - return { error: "Error: url is required for crawl mode." }; - } - const limit = resolveLimit(obj.limit, CRAWL_DEFAULT_LIMIT); - if ("error" in limit) { - return limit; - } - const result: CrawlArgs = { - mode: "crawl", - url: url.value, - limit: limit.value, - format: format.value, - }; - return result; - } - case "map": { - if (url.value === undefined || url.value.trim().length === 0) { - return { error: "Error: url is required for map mode." }; - } - const result: MapArgs = { - mode: "map", - url: url.value, - format: format.value, - }; - return result; - } - } + if (args === null || args === undefined || typeof args !== "object") { + return { error: "Error: Arguments must be an object." }; + } + const obj = args as Record<string, unknown>; + + const format = resolveFormat(obj.format); + if ("error" in format) { + return format; + } + + const mode = resolveMode(obj.mode, obj.query, obj.url); + if ("error" in mode) { + return mode; + } + + const query = optionalString(obj.query, "query"); + if ("error" in query) { + return query; + } + + const url = optionalString(obj.url, "url"); + if ("error" in url) { + return url; + } + + switch (mode.value) { + case "search": { + if (query.value === undefined || query.value.trim().length === 0) { + return { error: "Error: query is required for search mode." }; + } + const limit = resolveLimit(obj.limit, SEARCH_DEFAULT_LIMIT); + if ("error" in limit) { + return limit; + } + const scrape = resolveBoolean(obj.scrape, "scrape"); + if ("error" in scrape) { + return scrape; + } + const lang = optionalString(obj.lang, "lang"); + if ("error" in lang) { + return lang; + } + const country = optionalString(obj.country, "country"); + if ("error" in country) { + return country; + } + const result: SearchArgs = { + mode: "search", + query: query.value, + limit: limit.value, + scrape: scrape.value, + format: format.value, + ...(lang.value !== undefined ? { lang: lang.value } : {}), + ...(country.value !== undefined ? { country: country.value } : {}), + }; + return result; + } + case "scrape": { + if (url.value === undefined || url.value.trim().length === 0) { + return { error: "Error: url is required for scrape mode." }; + } + const result: ScrapeArgs = { + mode: "scrape", + url: url.value, + format: format.value, + }; + return result; + } + case "crawl": { + if (url.value === undefined || url.value.trim().length === 0) { + return { error: "Error: url is required for crawl mode." }; + } + const limit = resolveLimit(obj.limit, CRAWL_DEFAULT_LIMIT); + if ("error" in limit) { + return limit; + } + const result: CrawlArgs = { + mode: "crawl", + url: url.value, + limit: limit.value, + format: format.value, + }; + return result; + } + case "map": { + if (url.value === undefined || url.value.trim().length === 0) { + return { error: "Error: url is required for map mode." }; + } + const result: MapArgs = { + mode: "map", + url: url.value, + format: format.value, + }; + return result; + } + } } diff --git a/packages/tool-web-search/tsconfig.json b/packages/tool-web-search/tsconfig.json index ff99a43..44ed916 100644 --- a/packages/tool-web-search/tsconfig.json +++ b/packages/tool-web-search/tsconfig.json @@ -1,6 +1,6 @@ { - "extends": "../../tsconfig.base.json", - "compilerOptions": { "rootDir": "src", "outDir": "dist", "composite": true }, - "include": ["src/**/*.ts"], - "references": [{ "path": "../kernel" }] + "extends": "../../tsconfig.base.json", + "compilerOptions": { "rootDir": "src", "outDir": "dist", "composite": true }, + "include": ["src/**/*.ts"], + "references": [{ "path": "../kernel" }] } |
