summaryrefslogtreecommitdiffhomepage
path: root/packages/tool-web-search/src/client.ts
diff options
context:
space:
mode:
Diffstat (limited to 'packages/tool-web-search/src/client.ts')
-rw-r--r--packages/tool-web-search/src/client.ts368
1 files changed, 184 insertions, 184 deletions
diff --git a/packages/tool-web-search/src/client.ts b/packages/tool-web-search/src/client.ts
index 071ba97..cd137f5 100644
--- a/packages/tool-web-search/src/client.ts
+++ b/packages/tool-web-search/src/client.ts
@@ -17,97 +17,97 @@ export const CRAWL_POLL_MS = 2_000;
export const CRAWL_MAX_WAIT_MS = 5 * 60 * 1_000;
export interface SearchParams {
- readonly query: string;
- readonly limit: number;
- readonly lang?: string;
- readonly country?: string;
- readonly scrapeOptions?: {
- readonly formats: readonly string[];
- readonly onlyMainContent: boolean;
- };
+ readonly query: string;
+ readonly limit: number;
+ readonly lang?: string;
+ readonly country?: string;
+ readonly scrapeOptions?: {
+ readonly formats: readonly string[];
+ readonly onlyMainContent: boolean;
+ };
}
export interface ScrapeParams {
- readonly url: string;
- readonly formats: readonly string[];
+ readonly url: string;
+ readonly formats: readonly string[];
}
export interface CrawlParams {
- readonly url: string;
- readonly limit: number;
- readonly formats: readonly string[];
+ readonly url: string;
+ readonly limit: number;
+ readonly formats: readonly string[];
}
export interface FirecrawlClient {
- readonly search: (params: SearchParams, signal: AbortSignal) => Promise<readonly SearchHit[]>;
- readonly scrape: (params: ScrapeParams, signal: AbortSignal) => Promise<ScrapeResult>;
- readonly crawl: (params: CrawlParams, signal: AbortSignal) => Promise<readonly CrawlPage[]>;
- readonly map: (url: string, signal: AbortSignal) => Promise<readonly string[]>;
+ readonly search: (params: SearchParams, signal: AbortSignal) => Promise<readonly SearchHit[]>;
+ readonly scrape: (params: ScrapeParams, signal: AbortSignal) => Promise<ScrapeResult>;
+ readonly crawl: (params: CrawlParams, signal: AbortSignal) => Promise<readonly CrawlPage[]>;
+ readonly map: (url: string, signal: AbortSignal) => Promise<readonly string[]>;
}
export interface FirecrawlClientDeps {
- readonly baseUrl: string;
- readonly fetchFn: FetchLike;
- readonly timeoutMs?: number;
- readonly pollMs?: number;
- readonly maxWaitMs?: number;
- readonly now?: () => number;
- readonly sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
+ readonly baseUrl: string;
+ readonly fetchFn: FetchLike;
+ readonly timeoutMs?: number;
+ readonly pollMs?: number;
+ readonly maxWaitMs?: number;
+ readonly now?: () => number;
+ readonly sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
}
interface SearchResponse {
- readonly success: boolean;
- readonly data?: readonly SearchHit[];
- readonly error?: string;
+ readonly success: boolean;
+ readonly data?: readonly SearchHit[];
+ readonly error?: string;
}
interface ScrapeResponse {
- readonly success: boolean;
- readonly data?: {
- readonly markdown?: string;
- readonly metadata?: { readonly title?: string };
- };
- readonly error?: string;
+ readonly success: boolean;
+ readonly data?: {
+ readonly markdown?: string;
+ readonly metadata?: { readonly title?: string };
+ };
+ readonly error?: string;
}
interface CrawlStartResponse {
- readonly success: boolean;
- readonly url?: string;
- readonly error?: string;
+ readonly success: boolean;
+ readonly url?: string;
+ readonly error?: string;
}
interface CrawlStatusResponse {
- readonly status: string;
- readonly data?: readonly CrawlPage[];
- readonly error?: string;
+ readonly status: string;
+ readonly data?: readonly CrawlPage[];
+ readonly error?: string;
}
interface MapResponse {
- readonly success: boolean;
- readonly links?: readonly string[];
- readonly error?: string;
+ readonly success: boolean;
+ readonly links?: readonly string[];
+ readonly error?: string;
}
/** Default sleep: resolve after `ms`, reject on abort. */
async function defaultSleep(ms: number, signal: AbortSignal): Promise<void> {
- return new Promise<void>((resolve, reject) => {
- if (signal.aborted) {
- reject(new Error("Request aborted."));
- return;
- }
- let timer: ReturnType<typeof setTimeout> | undefined;
- const onAbort = (): void => {
- if (timer !== undefined) {
- clearTimeout(timer);
- }
- reject(new Error("Request aborted."));
- };
- timer = setTimeout(() => {
- signal.removeEventListener("abort", onAbort);
- resolve();
- }, ms);
- signal.addEventListener("abort", onAbort, { once: true });
- });
+ return new Promise<void>((resolve, reject) => {
+ if (signal.aborted) {
+ reject(new Error("Request aborted."));
+ return;
+ }
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const onAbort = (): void => {
+ if (timer !== undefined) {
+ clearTimeout(timer);
+ }
+ reject(new Error("Request aborted."));
+ };
+ timer = setTimeout(() => {
+ signal.removeEventListener("abort", onAbort);
+ resolve();
+ }, ms);
+ signal.addEventListener("abort", onAbort, { once: true });
+ });
}
/**
@@ -116,128 +116,128 @@ async function defaultSleep(ms: number, signal: AbortSignal): Promise<void> {
* is combined with the caller's cancellation signal via `AbortSignal.any`.
*/
export function createFirecrawlClient(deps: FirecrawlClientDeps): FirecrawlClient {
- const baseUrl = deps.baseUrl;
- const fetchFn = deps.fetchFn;
- const timeoutMs = deps.timeoutMs ?? DEFAULT_TIMEOUT_MS;
- const pollMs = deps.pollMs ?? CRAWL_POLL_MS;
- const maxWaitMs = deps.maxWaitMs ?? CRAWL_MAX_WAIT_MS;
- const now = deps.now ?? Date.now;
- const sleep = deps.sleep ?? defaultSleep;
-
- async function request(
- method: "POST" | "GET",
- url: string,
- body: unknown,
- signal: AbortSignal,
- ): Promise<unknown> {
- const controller = new AbortController();
- const timeout = setTimeout(() => controller.abort(), timeoutMs);
- const combined = AbortSignal.any([signal, controller.signal]);
- try {
- let response: Response;
- try {
- response = await fetchFn(url, {
- method,
- headers:
- body !== undefined
- ? { "Content-Type": "application/json", Accept: "application/json" }
- : { Accept: "application/json" },
- body: body !== undefined ? JSON.stringify(body) : undefined,
- signal: combined,
- });
- } catch (err) {
- if (signal.aborted) {
- throw new Error("Request aborted.");
- }
- if (controller.signal.aborted) {
- throw new Error(`Firecrawl request timed out after ${timeoutMs / 1000} seconds.`);
- }
- throw err;
- }
- if (!response.ok) {
- const text = await response.text().catch(() => "");
- throw new Error(`HTTP ${response.status} ${response.statusText}${text ? `: ${text}` : ""}`);
- }
- try {
- return await response.json();
- } catch {
- throw new Error("Failed to parse Firecrawl response as JSON");
- }
- } finally {
- clearTimeout(timeout);
- }
- }
-
- async function post(endpoint: string, body: unknown, signal: AbortSignal): Promise<unknown> {
- return request("POST", `${baseUrl}/${endpoint}`, body, signal);
- }
-
- return {
- async search(params: SearchParams, signal: AbortSignal): Promise<readonly SearchHit[]> {
- const body: Record<string, unknown> = { query: params.query, limit: params.limit };
- if (params.lang !== undefined) {
- body.lang = params.lang;
- }
- if (params.country !== undefined) {
- body.country = params.country;
- }
- if (params.scrapeOptions !== undefined) {
- body.scrapeOptions = params.scrapeOptions;
- }
- const json = (await post("search", body, signal)) as SearchResponse;
- if (!json.success) {
- throw new Error(json.error ?? "Unknown error");
- }
- return json.data ?? [];
- },
-
- async scrape(params: ScrapeParams, signal: AbortSignal): Promise<ScrapeResult> {
- const body = {
- url: params.url,
- formats: params.formats,
- onlyMainContent: true,
- };
- const json = (await post("scrape", body, signal)) as ScrapeResponse;
- if (!json.success) {
- throw new Error(json.error ?? "Unknown error");
- }
- return json;
- },
-
- async crawl(params: CrawlParams, signal: AbortSignal): Promise<readonly CrawlPage[]> {
- const body = {
- url: params.url,
- limit: params.limit,
- scrapeOptions: { formats: params.formats, onlyMainContent: true },
- };
- const startJson = (await post("crawl", body, signal)) as CrawlStartResponse;
- if (!startJson.success) {
- throw new Error(startJson.error ?? "Unknown error");
- }
- const statusUrl = startJson.url;
- if (statusUrl === undefined) {
- throw new Error("crawl response missing status URL.");
- }
- const started = now();
- while (now() - started < maxWaitMs) {
- await sleep(pollMs, signal);
- const status = (await request("GET", statusUrl, undefined, signal)) as CrawlStatusResponse;
- if (status.status === "completed") {
- return status.data ?? [];
- }
- if (status.status === "failed") {
- throw new Error(`crawl failed: ${status.error ?? "unknown"}`);
- }
- }
- throw new Error("crawl timed out waiting for completion.");
- },
-
- async map(url: string, signal: AbortSignal): Promise<readonly string[]> {
- const json = (await post("map", { url }, signal)) as MapResponse;
- if (!json.success) {
- throw new Error(json.error ?? "Unknown error");
- }
- return json.links ?? [];
- },
- };
+ const baseUrl = deps.baseUrl;
+ const fetchFn = deps.fetchFn;
+ const timeoutMs = deps.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+ const pollMs = deps.pollMs ?? CRAWL_POLL_MS;
+ const maxWaitMs = deps.maxWaitMs ?? CRAWL_MAX_WAIT_MS;
+ const now = deps.now ?? Date.now;
+ const sleep = deps.sleep ?? defaultSleep;
+
+ async function request(
+ method: "POST" | "GET",
+ url: string,
+ body: unknown,
+ signal: AbortSignal,
+ ): Promise<unknown> {
+ const controller = new AbortController();
+ const timeout = setTimeout(() => controller.abort(), timeoutMs);
+ const combined = AbortSignal.any([signal, controller.signal]);
+ try {
+ let response: Response;
+ try {
+ response = await fetchFn(url, {
+ method,
+ headers:
+ body !== undefined
+ ? { "Content-Type": "application/json", Accept: "application/json" }
+ : { Accept: "application/json" },
+ body: body !== undefined ? JSON.stringify(body) : undefined,
+ signal: combined,
+ });
+ } catch (err) {
+ if (signal.aborted) {
+ throw new Error("Request aborted.");
+ }
+ if (controller.signal.aborted) {
+ throw new Error(`Firecrawl request timed out after ${timeoutMs / 1000} seconds.`);
+ }
+ throw err;
+ }
+ if (!response.ok) {
+ const text = await response.text().catch(() => "");
+ throw new Error(`HTTP ${response.status} ${response.statusText}${text ? `: ${text}` : ""}`);
+ }
+ try {
+ return await response.json();
+ } catch {
+ throw new Error("Failed to parse Firecrawl response as JSON");
+ }
+ } finally {
+ clearTimeout(timeout);
+ }
+ }
+
+ async function post(endpoint: string, body: unknown, signal: AbortSignal): Promise<unknown> {
+ return request("POST", `${baseUrl}/${endpoint}`, body, signal);
+ }
+
+ return {
+ async search(params: SearchParams, signal: AbortSignal): Promise<readonly SearchHit[]> {
+ const body: Record<string, unknown> = { query: params.query, limit: params.limit };
+ if (params.lang !== undefined) {
+ body.lang = params.lang;
+ }
+ if (params.country !== undefined) {
+ body.country = params.country;
+ }
+ if (params.scrapeOptions !== undefined) {
+ body.scrapeOptions = params.scrapeOptions;
+ }
+ const json = (await post("search", body, signal)) as SearchResponse;
+ if (!json.success) {
+ throw new Error(json.error ?? "Unknown error");
+ }
+ return json.data ?? [];
+ },
+
+ async scrape(params: ScrapeParams, signal: AbortSignal): Promise<ScrapeResult> {
+ const body = {
+ url: params.url,
+ formats: params.formats,
+ onlyMainContent: true,
+ };
+ const json = (await post("scrape", body, signal)) as ScrapeResponse;
+ if (!json.success) {
+ throw new Error(json.error ?? "Unknown error");
+ }
+ return json;
+ },
+
+ async crawl(params: CrawlParams, signal: AbortSignal): Promise<readonly CrawlPage[]> {
+ const body = {
+ url: params.url,
+ limit: params.limit,
+ scrapeOptions: { formats: params.formats, onlyMainContent: true },
+ };
+ const startJson = (await post("crawl", body, signal)) as CrawlStartResponse;
+ if (!startJson.success) {
+ throw new Error(startJson.error ?? "Unknown error");
+ }
+ const statusUrl = startJson.url;
+ if (statusUrl === undefined) {
+ throw new Error("crawl response missing status URL.");
+ }
+ const started = now();
+ while (now() - started < maxWaitMs) {
+ await sleep(pollMs, signal);
+ const status = (await request("GET", statusUrl, undefined, signal)) as CrawlStatusResponse;
+ if (status.status === "completed") {
+ return status.data ?? [];
+ }
+ if (status.status === "failed") {
+ throw new Error(`crawl failed: ${status.error ?? "unknown"}`);
+ }
+ }
+ throw new Error("crawl timed out waiting for completion.");
+ },
+
+ async map(url: string, signal: AbortSignal): Promise<readonly string[]> {
+ const json = (await post("map", { url }, signal)) as MapResponse;
+ if (!json.success) {
+ throw new Error(json.error ?? "Unknown error");
+ }
+ return json.links ?? [];
+ },
+ };
}