Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'main' into v1/llm-extract

Browse Source
This commit is contained in:
Nicolas 2024-08-30 15:42:45 -03:00
commit 0732997a13
26 changed files with 949 additions and 1309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/controllers/auth.ts
View File

@ -104,7 +104,8 @@ export async function supaAuthenticateUser(
status?: number; status?: number;
plan?: PlanType; plan?: PlanType;
}> { }> {
const authHeader = req.headers.authorization;
const authHeader = req.headers.authorization ?? (req.headers["sec-websocket-protocol"] ? `Bearer ${req.headers["sec-websocket-protocol"]}` : null);
if (!authHeader) { if (!authHeader) {
return { success: false, error: "Unauthorized", status: 401 }; return { success: false, error: "Unauthorized", status: 401 };
} }

2
apps/api/src/controllers/v1/crawl-status-ws.ts
View File

@ -85,6 +85,8 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth<CrawlStatusPara
} }
} }
doneJobIDs.push(...newlyDoneJobIDs);
setTimeout(loop, 1000); setTimeout(loop, 1000);
}; };

3
apps/api/src/controllers/v1/crawl.ts
View File

@ -110,6 +110,7 @@ export async function crawlController(
origin: "api", origin: "api",
crawl_id: id, crawl_id: id,
sitemapped: true, sitemapped: true,
webhook: req.body.webhook,
v1: true, v1: true,
}, },
opts: { opts: {
@ -155,3 +156,5 @@ export async function crawlController(
url: `${req.protocol}://${req.get("host")}/v1/crawl/${id}`, url: `${req.protocol}://${req.get("host")}/v1/crawl/${id}`,
}); });
} }

2
apps/api/src/controllers/v1/types.ts
View File

@ -337,6 +337,8 @@ export function legacyExtractorOptions(x: ExtractOptions): ExtractorOptions {
} }
export function legacyDocumentConverter(doc: any): Document { export function legacyDocumentConverter(doc: any): Document {
if (doc === null || doc === undefined) return doc;
if (doc.metadata) { if (doc.metadata) {
if (doc.metadata.screenshot) { if (doc.metadata.screenshot) {
doc.screenshot = doc.metadata.screenshot; doc.screenshot = doc.metadata.screenshot;

22
apps/api/src/lib/supabase-jobs.ts
View File

@ -1,10 +1,12 @@
import { supabase_service } from "../services/supabase"; import { supabase_service } from "../services/supabase";
import { Logger } from "./logger";
import * as Sentry from "@sentry/node";
export const supabaseGetJobById = async (jobId: string) => { export const supabaseGetJobById = async (jobId: string) => {
const { data, error } = await supabase_service const { data, error } = await supabase_service
.from('firecrawl_jobs') .from("firecrawl_jobs")
.select('*') .select("*")
.eq('job_id', jobId) .eq("job_id", jobId)
.single(); .single();
if (error) { if (error) {
@ -16,15 +18,16 @@ export const supabaseGetJobById = async (jobId: string) => {
} }
return data; return data;
} };
export const supabaseGetJobsById = async (jobIds: string[]) => { export const supabaseGetJobsById = async (jobIds: string[]) => {
const { data, error } = await supabase_service const { data, error } = await supabase_service.rpc("get_jobs_by_ids", {
.from('firecrawl_jobs') job_ids: jobIds,
.select('*') });
.in('job_id', jobIds);
if (error) { if (error) {
Logger.error(`Error in get_jobs_by_ids: ${error}`);
Sentry.captureException(error);
return []; return [];
} }
@ -33,5 +36,4 @@ export const supabaseGetJobsById = async (jobIds: string[]) => {
} }
return data; return data;
} };

136
apps/js-sdk/README.md
View File

@ -1,136 +0,0 @@
# Firecrawl Node SDK
The Firecrawl Node SDK is a library that allows you to easily scrape and crawl websites, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for interacting with the Firecrawl API.
## Installation
To install the Firecrawl Node SDK, you can use npm:
```bash
npm install @mendable/firecrawl-js
```
## Usage
1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` class.
Here's an example of how to use the SDK with error handling:
```js
import FirecrawlApp, { CrawlParams, CrawlStatusResponse } from '@mendable/firecrawl-js';
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});
// Scrape a website
const scrapeResponse = await app.scrapeUrl('https://firecrawl.dev', {
formats: ['markdown', 'html'],
});
if (scrapeResponse) {
console.log(scrapeResponse)
}
// Crawl a website
const crawlResponse = await app.crawlUrl('https://firecrawl.dev', {
limit: 100,
scrapeOptions: {
formats: ['markdown', 'html'],
}
} as CrawlParams, true, 30) as CrawlStatusResponse;
if (crawlResponse) {
console.log(crawlResponse)
}
```
### Scraping a URL
To scrape a single URL with error handling, use the `scrapeUrl` method. It takes the URL as a parameter and returns the scraped data as a dictionary.
```js
const url = "https://example.com";
const scrapedData = await app.scrapeUrl(url);
```
### Crawling a Website
To crawl a website with error handling, use the `crawlUrl` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
```js
const crawlResponse = await app.crawlUrl('https://firecrawl.dev', {
limit: 100,
scrapeOptions: {
formats: ['markdown', 'html'],
}
} as CrawlParams, true, 30) as CrawlStatusResponse;
if (crawlResponse) {
console.log(crawlResponse)
}
```
### Checking Crawl Status
To check the status of a crawl job with error handling, use the `checkCrawlStatus` method. It takes the job ID as a parameter and returns the current status of the crawl job.
```js
const status = await app.checkCrawlStatus(id);
```
### Extracting structured data from a URL
With LLM extraction, you can easily extract structured data from any URL. We support zod schema to make it easier for you too. Here is how you to use it:
```js
import FirecrawlApp from "@mendable/firecrawl-js";
import { z } from "zod";
const app = new FirecrawlApp({
apiKey: "fc-YOUR_API_KEY",
});
// Define schema to extract contents into
const schema = z.object({
top: z
.array(
z.object({
title: z.string(),
points: z.number(),
by: z.string(),
commentsURL: z.string(),
})
)
.length(5)
.describe("Top 5 stories on Hacker News"),
});
const scrapeResult = await app.scrapeUrl("https://firecrawl.dev", {
extractorOptions: { extractionSchema: schema },
});
console.log(scrapeResult.data["llm_extraction"]);
```
### Map a Website
Use `map_url` to generate a list of URLs from a website. The `params` argument let you customize the mapping process, including options to exclude subdomains or to utilize the sitemap.
```js
const mapResult = await app.mapUrl('https://example.com') as MapResponse;
console.log(mapResult)
```
## Error Handling
The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. The examples above demonstrate how to handle these errors using `try/catch` blocks.
## License
The Firecrawl Node SDK is licensed under the MIT License. This means you are free to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the SDK, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Please note that while this SDK is MIT licensed, it is part of a larger project which may be under different licensing terms. Always refer to the license information in the root directory of the main project for overall licensing details.

79
apps/js-sdk/example.js
View File

@ -1,33 +1,62 @@
import FirecrawlApp from './firecrawl/src/index'; //'@mendable/firecrawl-js'; import FirecrawlApp from '@mendable/firecrawl-js';
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});
// Scrape a website: const main = async () => {
const scrapeResult = await app.scrapeUrl('firecrawl.dev');
if (scrapeResult.data) { // Scrape a website:
console.log(scrapeResult.data.markdown) const scrapeResult = await app.scrapeUrl('firecrawl.dev');
}
// Crawl a website: if (scrapeResult.success) {
const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false); console.log(scrapeResult.markdown)
console.log(crawlResult)
const jobId = await crawlResult['jobId'];
console.log(jobId);
let job;
while (true) {
job = await app.checkCrawlStatus(jobId);
if (job.status === 'completed') {
break;
} }
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
// Crawl a website:
const crawlResult = await app.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5});
console.log(crawlResult);
// Asynchronously crawl a website:
const asyncCrawlResult = await app.asyncCrawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5});
if (asyncCrawlResult.success) {
const id = asyncCrawlResult.id;
console.log(id);
let checkStatus;
if (asyncCrawlResult.success) {
while (true) {
checkStatus = await app.checkCrawlStatus(id);
if (checkStatus.success && checkStatus.status === 'completed') {
break;
}
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
}
if (checkStatus.success && checkStatus.data) {
console.log(checkStatus.data[0].markdown);
}
}
}
// Map a website:
const mapResult = await app.mapUrl('https://firecrawl.dev');
console.log(mapResult)
// Crawl a website with WebSockets:
const watch = await app.crawlUrlAndWatch('mendable.ai', { excludePaths: ['blog/*'], limit: 5});
watch.addEventListener("document", doc => {
console.log("DOC", doc.detail);
});
watch.addEventListener("error", err => {
console.error("ERR", err.detail.error);
});
watch.addEventListener("done", state => {
console.log("DONE", state.detail.status);
});
} }
if (job.data) { main()
console.log(job.data[0].markdown);
}
const mapResult = await app.map('https://firecrawl.dev');
console.log(mapResult)

56
apps/js-sdk/example.ts
View File

@ -1,4 +1,4 @@
import FirecrawlApp, { CrawlStatusResponse, CrawlResponse } from '@mendable/firecrawl-js'; import FirecrawlApp, { CrawlStatusResponse, ErrorResponse } from '@mendable/firecrawl-js';
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});
@ -7,33 +7,55 @@ const main = async () => {
// Scrape a website: // Scrape a website:
const scrapeResult = await app.scrapeUrl('firecrawl.dev'); const scrapeResult = await app.scrapeUrl('firecrawl.dev');
if (scrapeResult) { if (scrapeResult.success) {
console.log(scrapeResult.markdown) console.log(scrapeResult.markdown)
} }
// Crawl a website: // Crawl a website:
// @ts-ignore const crawlResult = await app.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5});
const crawlResult = await app.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}, false) as CrawlResponse; console.log(crawlResult);
console.log(crawlResult)
const id = crawlResult.id; // Asynchronously crawl a website:
console.log(id); const asyncCrawlResult = await app.asyncCrawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5});
if (asyncCrawlResult.success) {
const id = asyncCrawlResult.id;
console.log(id);
let checkStatus: CrawlStatusResponse; let checkStatus: CrawlStatusResponse | ErrorResponse;
while (true) { if (asyncCrawlResult.success) {
checkStatus = await app.checkCrawlStatus(id); while (true) {
if (checkStatus.status === 'completed') { checkStatus = await app.checkCrawlStatus(id);
break; if (checkStatus.success && checkStatus.status === 'completed') {
break;
}
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
}
if (checkStatus.success && checkStatus.data) {
console.log(checkStatus.data[0].markdown);
}
} }
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
}
if (checkStatus.data) {
console.log(checkStatus.data[0].markdown);
} }
// Map a website:
const mapResult = await app.mapUrl('https://firecrawl.dev'); const mapResult = await app.mapUrl('https://firecrawl.dev');
console.log(mapResult) console.log(mapResult)
// Crawl a website with WebSockets:
const watch = await app.crawlUrlAndWatch('mendable.ai', { excludePaths: ['blog/*'], limit: 5});
watch.addEventListener("document", doc => {
console.log("DOC", doc.detail);
});
watch.addEventListener("error", err => {
console.error("ERR", err.detail.error);
});
watch.addEventListener("done", state => {
console.log("DONE", state.detail.status);
});
} }
main() main()

85
apps/js-sdk/exampleV0.js
View File

@ -1,85 +0,0 @@
import { v4 as uuidv4 } from 'uuid';
import FirecrawlApp from '@mendable/firecrawl-js';
import { z } from "zod";
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});
// Scrape a website:
const scrapeResult = await app.scrapeUrl('firecrawl.dev');
console.log(scrapeResult.data.content)
// Crawl a website:
const idempotencyKey = uuidv4(); // optional
const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false, 2, idempotencyKey);
console.log(crawlResult)
const jobId = await crawlResult['jobId'];
console.log(jobId);
let job;
while (true) {
job = await app.checkCrawlStatus(jobId);
if (job.status == 'completed') {
break;
}
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
}
console.log(job.data[0].content);
// Search for a query:
const query = 'what is mendable?'
const searchResult = await app.search(query)
console.log(searchResult)
// LLM Extraction:
// Define schema to extract contents into using zod schema
const zodSchema = z.object({
top: z
.array(
z.object({
title: z.string(),
points: z.number(),
by: z.string(),
commentsURL: z.string(),
})
)
.length(5)
.describe("Top 5 stories on Hacker News"),
});
let llmExtractionResult = await app.scrapeUrl("https://news.ycombinator.com", {
extractorOptions: { extractionSchema: zodSchema },
});
console.log(llmExtractionResult.data.llm_extraction);
// Define schema to extract contents into using json schema
const jsonSchema = {
"type": "object",
"properties": {
"top": {
"type": "array",
"items": {
"type": "object",
"properties": {
"title": {"type": "string"},
"points": {"type": "number"},
"by": {"type": "string"},
"commentsURL": {"type": "string"}
},
"required": ["title", "points", "by", "commentsURL"]
},
"minItems": 5,
"maxItems": 5,
"description": "Top 5 stories on Hacker News"
}
},
"required": ["top"]
}
llmExtractionResult = await app.scrapeUrl("https://news.ycombinator.com", {
extractorOptions: { extractionSchema: jsonSchema },
});
console.log(llmExtractionResult.data.llm_extraction);

93
apps/js-sdk/exampleV0.ts
View File

@ -1,93 +0,0 @@
import FirecrawlApp, { ScrapeResponseV0, CrawlStatusResponseV0, SearchResponseV0 } from './firecrawl/src/index' //'@mendable/firecrawl-js';
import { z } from "zod";
const app = new FirecrawlApp<"v0">({apiKey: "fc-YOUR_API_KEY", version: "v0"})
// Scrape a website:
const scrapeResult = await app.scrapeUrl('firecrawl.dev');
if (scrapeResult.data) {
console.log(scrapeResult.data.content)
}
// Crawl a website:
const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false);
console.log(crawlResult)
const jobId: string = await crawlResult['jobId'];
console.log(jobId);
let job: CrawlStatusResponseV0;
while (true) {
job = await app.checkCrawlStatus(jobId) as CrawlStatusResponseV0;
if (job.status === 'completed') {
break;
}
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
}
if (job.data) {
console.log(job.data[0].content);
}
// Search for a query:
const query = 'what is mendable?'
const searchResult = await app.search(query) as SearchResponseV0;
if (searchResult.data) {
console.log(searchResult.data[0].content)
}
// LLM Extraction:
// Define schema to extract contents into using zod schema
const zodSchema = z.object({
top: z
.array(
z.object({
title: z.string(),
points: z.number(),
by: z.string(),
commentsURL: z.string(),
})
)
.length(5)
.describe("Top 5 stories on Hacker News"),
});
let llmExtractionResult = await app.scrapeUrl("https://news.ycombinator.com");
if (llmExtractionResult.data) {
console.log(llmExtractionResult.data[0].llm_extraction);
}
// Define schema to extract contents into using json schema
const jsonSchema = {
"type": "object",
"properties": {
"top": {
"type": "array",
"items": {
"type": "object",
"properties": {
"title": {"type": "string"},
"points": {"type": "number"},
"by": {"type": "string"},
"commentsURL": {"type": "string"}
},
"required": ["title", "points", "by", "commentsURL"]
},
"minItems": 5,
"maxItems": 5,
"description": "Top 5 stories on Hacker News"
}
},
"required": ["top"]
}
llmExtractionResult = await app.scrapeUrl("https://news.ycombinator.com", {
extractorOptions: { extractionSchema: jsonSchema },
});
if (llmExtractionResult.data) {
console.log(llmExtractionResult.data[0].llm_extraction);
}

42
apps/js-sdk/firecrawl/README.md
View File

@ -37,11 +37,9 @@ const crawlResponse = await app.crawlUrl('https://firecrawl.dev', {
scrapeOptions: { scrapeOptions: {
formats: ['markdown', 'html'], formats: ['markdown', 'html'],
} }
} as CrawlParams, true, 30) as CrawlStatusResponse; })
if (crawlResponse) { console.log(crawlResponse)
console.log(crawlResponse)
}
``` ```
### Scraping a URL ### Scraping a URL
@ -63,16 +61,21 @@ const crawlResponse = await app.crawlUrl('https://firecrawl.dev', {
scrapeOptions: { scrapeOptions: {
formats: ['markdown', 'html'], formats: ['markdown', 'html'],
} }
} as CrawlParams, true, 30) as CrawlStatusResponse; })
```
if (crawlResponse) {
console.log(crawlResponse) ### Asynchronous Crawl
}
To initiate an asynchronous crawl of a website, utilize the AsyncCrawlURL method. This method requires the starting URL and optional parameters as inputs. The params argument enables you to define various settings for the asynchronous crawl, such as the maximum number of pages to crawl, permitted domains, and the output format. Upon successful initiation, this method returns an ID, which is essential for subsequently checking the status of the crawl.
```js
const asyncCrawlResult = await app.asyncCrawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5});
``` ```
### Checking Crawl Status ### Checking Crawl Status
To check the status of a crawl job with error handling, use the `checkCrawlStatus` method. It takes the job ID as a parameter and returns the current status of the crawl job. To check the status of a crawl job with error handling, use the `checkCrawlStatus` method. It takes the job ID as a parameter and returns the current status of the crawl job`
```js ```js
const status = await app.checkCrawlStatus(id); const status = await app.checkCrawlStatus(id);
@ -121,6 +124,27 @@ const mapResult = await app.mapUrl('https://example.com') as MapResponse;
console.log(mapResult) console.log(mapResult)
``` ```
### Crawl a website with WebSockets
To crawl a website with WebSockets, use the `crawlUrlAndWatch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
```js
// Crawl a website with WebSockets:
const watch = await app.crawlUrlAndWatch('mendable.ai', { excludePaths: ['blog/*'], limit: 5});
watch.addEventListener("document", doc => {
console.log("DOC", doc.detail);
});
watch.addEventListener("error", err => {
console.error("ERR", err.detail.error);
});
watch.addEventListener("done", state => {
console.log("DONE", state.detail.status);
});
```
## Error Handling ## Error Handling
The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. The examples above demonstrate how to handle these errors using `try/catch` blocks. The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. The examples above demonstrate how to handle these errors using `try/catch` blocks.

257
apps/js-sdk/firecrawl/build/cjs/index.js
View File

@ -3,9 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod }; return (mod && mod.__esModule) ? mod : { "default": mod };
}; };
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
exports.CrawlWatcher = void 0;
const axios_1 = __importDefault(require("axios")); const axios_1 = __importDefault(require("axios"));
const zod_1 = require("zod"); const zod_1 = require("zod");
const zod_to_json_schema_1 = require("zod-to-json-schema"); const zod_to_json_schema_1 = require("zod-to-json-schema");
const isows_1 = require("isows");
const typescript_event_target_1 = require("typescript-event-target");
/** /**
* Main class for interacting with the Firecrawl API. * Main class for interacting with the Firecrawl API.
* Provides methods for scraping, searching, crawling, and mapping web content. * Provides methods for scraping, searching, crawling, and mapping web content.
@ -15,13 +18,9 @@ class FirecrawlApp {
* Initializes a new instance of the FirecrawlApp class. * Initializes a new instance of the FirecrawlApp class.
* @param config - Configuration options for the FirecrawlApp instance. * @param config - Configuration options for the FirecrawlApp instance.
*/ */
constructor({ apiKey = null, apiUrl = null, version = "v1" }) { constructor({ apiKey = null, apiUrl = null }) {
this.apiKey = apiKey || ""; this.apiKey = apiKey || "";
this.apiUrl = apiUrl || "https://api.firecrawl.dev"; this.apiUrl = apiUrl || "https://api.firecrawl.dev";
this.version = version;
if (!this.apiKey) {
throw new Error("No API key provided");
}
} }
/** /**
* Scrapes a URL using the Firecrawl API. * Scrapes a URL using the Firecrawl API.
@ -65,16 +64,16 @@ class FirecrawlApp {
}; };
} }
try { try {
const response = await axios_1.default.post(this.apiUrl + `/${this.version}/scrape`, jsonData, { headers }); const response = await axios_1.default.post(this.apiUrl + `/v1/scrape`, jsonData, { headers });
if (response.status === 200) { if (response.status === 200) {
const responseData = response.data; const responseData = response.data;
if (responseData.success) { if (responseData.success) {
return (this.version === 'v0' ? responseData : { return {
success: true, success: true,
warning: responseData.warning, warning: responseData.warning,
error: responseData.error, error: responseData.error,
...responseData.data ...responseData.data
}); };
} }
else { else {
throw new Error(`Failed to scrape URL. Error: ${responseData.error}`); throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
@ -90,80 +89,52 @@ class FirecrawlApp {
return { success: false, error: "Internal server error." }; return { success: false, error: "Internal server error." };
} }
/** /**
* Searches for a query using the Firecrawl API. * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
* @param query - The query to search for. * @param query - The search query string.
* @param params - Additional parameters for the search request. * @param params - Additional parameters for the search.
* @returns The response from the search operation. * @returns Throws an error advising to use version 0 of the API.
*/ */
async search(query, params) { async search(query, params) {
if (this.version === "v1") { throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
}
const headers = {
"Content-Type": "application/json",
Authorization: `Bearer ${this.apiKey}`,
};
let jsonData = { query };
if (params) {
jsonData = { ...jsonData, ...params };
}
try {
const response = await axios_1.default.post(this.apiUrl + "/v0/search", jsonData, { headers });
if (response.status === 200) {
const responseData = response.data;
if (responseData.success) {
return responseData;
}
else {
throw new Error(`Failed to search. Error: ${responseData.error}`);
}
}
else {
this.handleError(response, "search");
}
}
catch (error) {
throw new Error(error.message);
}
return { success: false, error: "Internal server error." };
} }
/** /**
* Initiates a crawl job for a URL using the Firecrawl API. * Initiates a crawl job for a URL using the Firecrawl API.
* @param url - The URL to crawl. * @param url - The URL to crawl.
* @param params - Additional parameters for the crawl request. * @param params - Additional parameters for the crawl request.
* @param waitUntilDone - Whether to wait for the crawl job to complete.
* @param pollInterval - Time in seconds for job status checks. * @param pollInterval - Time in seconds for job status checks.
* @param idempotencyKey - Optional idempotency key for the request. * @param idempotencyKey - Optional idempotency key for the request.
* @returns The response from the crawl operation. * @returns The response from the crawl operation.
*/ */
async crawlUrl(url, params, waitUntilDone = true, pollInterval = 2, idempotencyKey) { async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
const headers = this.prepareHeaders(idempotencyKey); const headers = this.prepareHeaders(idempotencyKey);
let jsonData = { url, ...params }; let jsonData = { url, ...params };
try { try {
const response = await this.postRequest(this.apiUrl + `/${this.version}/crawl`, jsonData, headers); const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers);
if (response.status === 200) { if (response.status === 200) {
const id = this.version === 'v0' ? response.data.jobId : response.data.id; const id = response.data.id;
let checkUrl = undefined; return this.monitorJobStatus(id, headers, pollInterval);
if (waitUntilDone) { }
if (this.version === 'v1') { else {
checkUrl = response.data.url; this.handleError(response, "start crawl job");
} }
return this.monitorJobStatus(id, headers, pollInterval, checkUrl); }
} catch (error) {
else { if (error.response?.data?.error) {
if (this.version === 'v0') { throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`);
return { }
success: true, else {
jobId: id throw new Error(error.message);
}; }
} }
else { return { success: false, error: "Internal server error." };
return { }
success: true, async asyncCrawlUrl(url, params, idempotencyKey) {
id: id const headers = this.prepareHeaders(idempotencyKey);
}; let jsonData = { url, ...params };
} try {
} const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers);
if (response.status === 200) {
return response.data;
} }
else { else {
this.handleError(response, "start crawl job"); this.handleError(response, "start crawl job");
@ -190,37 +161,19 @@ class FirecrawlApp {
} }
const headers = this.prepareHeaders(); const headers = this.prepareHeaders();
try { try {
const response = await this.getRequest(this.version === 'v1' ? const response = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
`${this.apiUrl}/${this.version}/crawl/${id}` :
`${this.apiUrl}/${this.version}/crawl/status/${id}`, headers);
if (response.status === 200) { if (response.status === 200) {
if (this.version === 'v0') { return ({
return { success: true,
success: true, status: response.data.status,
status: response.data.status, total: response.data.total,
current: response.data.current, completed: response.data.completed,
current_url: response.data.current_url, creditsUsed: response.data.creditsUsed,
current_step: response.data.current_step, expiresAt: new Date(response.data.expiresAt),
total: response.data.total, next: response.data.next,
data: response.data.data, data: response.data.data,
partial_data: !response.data.data error: response.data.error
? response.data.partial_data });
: undefined,
};
}
else {
return {
success: true,
status: response.data.status,
total: response.data.total,
completed: response.data.completed,
creditsUsed: response.data.creditsUsed,
expiresAt: new Date(response.data.expiresAt),
next: response.data.next,
data: response.data.data,
error: response.data.error
};
}
} }
else { else {
this.handleError(response, "check crawl status"); this.handleError(response, "check crawl status");
@ -229,29 +182,21 @@ class FirecrawlApp {
catch (error) { catch (error) {
throw new Error(error.message); throw new Error(error.message);
} }
return this.version === 'v0' ? return { success: false, error: "Internal server error." };
{ }
success: false, async crawlUrlAndWatch(url, params, idempotencyKey) {
status: "unknown", const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
current: 0, if (crawl.success && crawl.id) {
current_url: "", const id = crawl.id;
current_step: "", return new CrawlWatcher(id, this);
total: 0, }
error: "Internal server error.", throw new Error("Crawl job failed to start");
} :
{
success: false,
error: "Internal server error.",
};
} }
async mapUrl(url, params) { async mapUrl(url, params) {
if (this.version == 'v0') {
throw new Error("Map is not supported in v0");
}
const headers = this.prepareHeaders(); const headers = this.prepareHeaders();
let jsonData = { url, ...params }; let jsonData = { url, ...params };
try { try {
const response = await this.postRequest(this.apiUrl + `/${this.version}/map`, jsonData, headers); const response = await this.postRequest(this.apiUrl + `/v1/map`, jsonData, headers);
if (response.status === 200) { if (response.status === 200) {
return response.data; return response.data;
} }
@ -303,21 +248,14 @@ class FirecrawlApp {
* @param checkUrl - Optional URL to check the status (used for v1 API) * @param checkUrl - Optional URL to check the status (used for v1 API)
* @returns The final job status or data. * @returns The final job status or data.
*/ */
async monitorJobStatus(id, headers, checkInterval, checkUrl) { async monitorJobStatus(id, headers, checkInterval) {
let apiUrl = '';
while (true) { while (true) {
if (this.version === 'v1') { const statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
apiUrl = checkUrl ?? `${this.apiUrl}/v1/crawl/${id}`;
}
else if (this.version === 'v0') {
apiUrl = `${this.apiUrl}/v0/crawl/status/${id}`;
}
const statusResponse = await this.getRequest(apiUrl, headers);
if (statusResponse.status === 200) { if (statusResponse.status === 200) {
const statusData = statusResponse.data; const statusData = statusResponse.data;
if (statusData.status === "completed") { if (statusData.status === "completed") {
if ("data" in statusData) { if ("data" in statusData) {
return this.version === 'v0' ? statusData.data : statusData; return statusData;
} }
else { else {
throw new Error("Crawl job completed but no data was returned"); throw new Error("Crawl job completed but no data was returned");
@ -352,3 +290,72 @@ class FirecrawlApp {
} }
} }
exports.default = FirecrawlApp; exports.default = FirecrawlApp;
class CrawlWatcher extends typescript_event_target_1.TypedEventTarget {
constructor(id, app) {
super();
this.ws = new isows_1.WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
this.status = "scraping";
this.data = [];
const messageHandler = (msg) => {
if (msg.type === "done") {
this.status = "completed";
this.dispatchTypedEvent("done", new CustomEvent("done", {
detail: {
status: this.status,
data: this.data,
},
}));
}
else if (msg.type === "error") {
this.status = "failed";
this.dispatchTypedEvent("error", new CustomEvent("error", {
detail: {
status: this.status,
data: this.data,
error: msg.error,
},
}));
}
else if (msg.type === "catchup") {
this.status = msg.data.status;
this.data.push(...(msg.data.data ?? []));
for (const doc of this.data) {
this.dispatchTypedEvent("document", new CustomEvent("document", {
detail: doc,
}));
}
}
else if (msg.type === "document") {
this.dispatchTypedEvent("document", new CustomEvent("document", {
detail: msg.data,
}));
}
};
this.ws.onmessage = ((ev) => {
if (typeof ev.data !== "string") {
this.ws.close();
return;
}
const msg = JSON.parse(ev.data);
messageHandler(msg);
}).bind(this);
this.ws.onclose = ((ev) => {
const msg = JSON.parse(ev.reason);
messageHandler(msg);
}).bind(this);
this.ws.onerror = ((_) => {
this.status = "failed";
this.dispatchTypedEvent("error", new CustomEvent("error", {
detail: {
status: this.status,
data: this.data,
error: "WebSocket error",
},
}));
}).bind(this);
}
close() {
this.ws.close();
}
}
exports.CrawlWatcher = CrawlWatcher;

255
apps/js-sdk/firecrawl/build/esm/index.js
View File

@ -1,6 +1,8 @@
import axios from "axios"; import axios from "axios";
import { z } from "zod"; import { z } from "zod";
import { zodToJsonSchema } from "zod-to-json-schema"; import { zodToJsonSchema } from "zod-to-json-schema";
import { WebSocket } from "isows";
import { TypedEventTarget } from "typescript-event-target";
/** /**
* Main class for interacting with the Firecrawl API. * Main class for interacting with the Firecrawl API.
* Provides methods for scraping, searching, crawling, and mapping web content. * Provides methods for scraping, searching, crawling, and mapping web content.
@ -10,13 +12,9 @@ export default class FirecrawlApp {
* Initializes a new instance of the FirecrawlApp class. * Initializes a new instance of the FirecrawlApp class.
* @param config - Configuration options for the FirecrawlApp instance. * @param config - Configuration options for the FirecrawlApp instance.
*/ */
constructor({ apiKey = null, apiUrl = null, version = "v1" }) { constructor({ apiKey = null, apiUrl = null }) {
this.apiKey = apiKey || ""; this.apiKey = apiKey || "";
this.apiUrl = apiUrl || "https://api.firecrawl.dev"; this.apiUrl = apiUrl || "https://api.firecrawl.dev";
this.version = version;
if (!this.apiKey) {
throw new Error("No API key provided");
}
} }
/** /**
* Scrapes a URL using the Firecrawl API. * Scrapes a URL using the Firecrawl API.
@ -60,16 +58,16 @@ export default class FirecrawlApp {
}; };
} }
try { try {
const response = await axios.post(this.apiUrl + `/${this.version}/scrape`, jsonData, { headers }); const response = await axios.post(this.apiUrl + `/v1/scrape`, jsonData, { headers });
if (response.status === 200) { if (response.status === 200) {
const responseData = response.data; const responseData = response.data;
if (responseData.success) { if (responseData.success) {
return (this.version === 'v0' ? responseData : { return {
success: true, success: true,
warning: responseData.warning, warning: responseData.warning,
error: responseData.error, error: responseData.error,
...responseData.data ...responseData.data
}); };
} }
else { else {
throw new Error(`Failed to scrape URL. Error: ${responseData.error}`); throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
@ -85,80 +83,52 @@ export default class FirecrawlApp {
return { success: false, error: "Internal server error." }; return { success: false, error: "Internal server error." };
} }
/** /**
* Searches for a query using the Firecrawl API. * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
* @param query - The query to search for. * @param query - The search query string.
* @param params - Additional parameters for the search request. * @param params - Additional parameters for the search.
* @returns The response from the search operation. * @returns Throws an error advising to use version 0 of the API.
*/ */
async search(query, params) { async search(query, params) {
if (this.version === "v1") { throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
}
const headers = {
"Content-Type": "application/json",
Authorization: `Bearer ${this.apiKey}`,
};
let jsonData = { query };
if (params) {
jsonData = { ...jsonData, ...params };
}
try {
const response = await axios.post(this.apiUrl + "/v0/search", jsonData, { headers });
if (response.status === 200) {
const responseData = response.data;
if (responseData.success) {
return responseData;
}
else {
throw new Error(`Failed to search. Error: ${responseData.error}`);
}
}
else {
this.handleError(response, "search");
}
}
catch (error) {
throw new Error(error.message);
}
return { success: false, error: "Internal server error." };
} }
/** /**
* Initiates a crawl job for a URL using the Firecrawl API. * Initiates a crawl job for a URL using the Firecrawl API.
* @param url - The URL to crawl. * @param url - The URL to crawl.
* @param params - Additional parameters for the crawl request. * @param params - Additional parameters for the crawl request.
* @param waitUntilDone - Whether to wait for the crawl job to complete.
* @param pollInterval - Time in seconds for job status checks. * @param pollInterval - Time in seconds for job status checks.
* @param idempotencyKey - Optional idempotency key for the request. * @param idempotencyKey - Optional idempotency key for the request.
* @returns The response from the crawl operation. * @returns The response from the crawl operation.
*/ */
async crawlUrl(url, params, waitUntilDone = true, pollInterval = 2, idempotencyKey) { async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
const headers = this.prepareHeaders(idempotencyKey); const headers = this.prepareHeaders(idempotencyKey);
let jsonData = { url, ...params }; let jsonData = { url, ...params };
try { try {
const response = await this.postRequest(this.apiUrl + `/${this.version}/crawl`, jsonData, headers); const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers);
if (response.status === 200) { if (response.status === 200) {
const id = this.version === 'v0' ? response.data.jobId : response.data.id; const id = response.data.id;
let checkUrl = undefined; return this.monitorJobStatus(id, headers, pollInterval);
if (waitUntilDone) { }
if (this.version === 'v1') { else {
checkUrl = response.data.url; this.handleError(response, "start crawl job");
} }
return this.monitorJobStatus(id, headers, pollInterval, checkUrl); }
} catch (error) {
else { if (error.response?.data?.error) {
if (this.version === 'v0') { throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`);
return { }
success: true, else {
jobId: id throw new Error(error.message);
}; }
} }
else { return { success: false, error: "Internal server error." };
return { }
success: true, async asyncCrawlUrl(url, params, idempotencyKey) {
id: id const headers = this.prepareHeaders(idempotencyKey);
}; let jsonData = { url, ...params };
} try {
} const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers);
if (response.status === 200) {
return response.data;
} }
else { else {
this.handleError(response, "start crawl job"); this.handleError(response, "start crawl job");
@ -185,37 +155,19 @@ export default class FirecrawlApp {
} }
const headers = this.prepareHeaders(); const headers = this.prepareHeaders();
try { try {
const response = await this.getRequest(this.version === 'v1' ? const response = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
`${this.apiUrl}/${this.version}/crawl/${id}` :
`${this.apiUrl}/${this.version}/crawl/status/${id}`, headers);
if (response.status === 200) { if (response.status === 200) {
if (this.version === 'v0') { return ({
return { success: true,
success: true, status: response.data.status,
status: response.data.status, total: response.data.total,
current: response.data.current, completed: response.data.completed,
current_url: response.data.current_url, creditsUsed: response.data.creditsUsed,
current_step: response.data.current_step, expiresAt: new Date(response.data.expiresAt),
total: response.data.total, next: response.data.next,
data: response.data.data, data: response.data.data,
partial_data: !response.data.data error: response.data.error
? response.data.partial_data });
: undefined,
};
}
else {
return {
success: true,
status: response.data.status,
total: response.data.total,
completed: response.data.completed,
creditsUsed: response.data.creditsUsed,
expiresAt: new Date(response.data.expiresAt),
next: response.data.next,
data: response.data.data,
error: response.data.error
};
}
} }
else { else {
this.handleError(response, "check crawl status"); this.handleError(response, "check crawl status");
@ -224,29 +176,21 @@ export default class FirecrawlApp {
catch (error) { catch (error) {
throw new Error(error.message); throw new Error(error.message);
} }
return this.version === 'v0' ? return { success: false, error: "Internal server error." };
{ }
success: false, async crawlUrlAndWatch(url, params, idempotencyKey) {
status: "unknown", const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
current: 0, if (crawl.success && crawl.id) {
current_url: "", const id = crawl.id;
current_step: "", return new CrawlWatcher(id, this);
total: 0, }
error: "Internal server error.", throw new Error("Crawl job failed to start");
} :
{
success: false,
error: "Internal server error.",
};
} }
async mapUrl(url, params) { async mapUrl(url, params) {
if (this.version == 'v0') {
throw new Error("Map is not supported in v0");
}
const headers = this.prepareHeaders(); const headers = this.prepareHeaders();
let jsonData = { url, ...params }; let jsonData = { url, ...params };
try { try {
const response = await this.postRequest(this.apiUrl + `/${this.version}/map`, jsonData, headers); const response = await this.postRequest(this.apiUrl + `/v1/map`, jsonData, headers);
if (response.status === 200) { if (response.status === 200) {
return response.data; return response.data;
} }
@ -298,21 +242,14 @@ export default class FirecrawlApp {
* @param checkUrl - Optional URL to check the status (used for v1 API) * @param checkUrl - Optional URL to check the status (used for v1 API)
* @returns The final job status or data. * @returns The final job status or data.
*/ */
async monitorJobStatus(id, headers, checkInterval, checkUrl) { async monitorJobStatus(id, headers, checkInterval) {
let apiUrl = '';
while (true) { while (true) {
if (this.version === 'v1') { const statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
apiUrl = checkUrl ?? `${this.apiUrl}/v1/crawl/${id}`;
}
else if (this.version === 'v0') {
apiUrl = `${this.apiUrl}/v0/crawl/status/${id}`;
}
const statusResponse = await this.getRequest(apiUrl, headers);
if (statusResponse.status === 200) { if (statusResponse.status === 200) {
const statusData = statusResponse.data; const statusData = statusResponse.data;
if (statusData.status === "completed") { if (statusData.status === "completed") {
if ("data" in statusData) { if ("data" in statusData) {
return this.version === 'v0' ? statusData.data : statusData; return statusData;
} }
else { else {
throw new Error("Crawl job completed but no data was returned"); throw new Error("Crawl job completed but no data was returned");
@ -346,3 +283,71 @@ export default class FirecrawlApp {
} }
} }
} }
export class CrawlWatcher extends TypedEventTarget {
constructor(id, app) {
super();
this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
this.status = "scraping";
this.data = [];
const messageHandler = (msg) => {
if (msg.type === "done") {
this.status = "completed";
this.dispatchTypedEvent("done", new CustomEvent("done", {
detail: {
status: this.status,
data: this.data,
},
}));
}
else if (msg.type === "error") {
this.status = "failed";
this.dispatchTypedEvent("error", new CustomEvent("error", {
detail: {
status: this.status,
data: this.data,
error: msg.error,
},
}));
}
else if (msg.type === "catchup") {
this.status = msg.data.status;
this.data.push(...(msg.data.data ?? []));
for (const doc of this.data) {
this.dispatchTypedEvent("document", new CustomEvent("document", {
detail: doc,
}));
}
}
else if (msg.type === "document") {
this.dispatchTypedEvent("document", new CustomEvent("document", {
detail: msg.data,
}));
}
};
this.ws.onmessage = ((ev) => {
if (typeof ev.data !== "string") {
this.ws.close();
return;
}
const msg = JSON.parse(ev.data);
messageHandler(msg);
}).bind(this);
this.ws.onclose = ((ev) => {
const msg = JSON.parse(ev.reason);
messageHandler(msg);
}).bind(this);
this.ws.onerror = ((_) => {
this.status = "failed";
this.dispatchTypedEvent("error", new CustomEvent("error", {
detail: {
status: this.status,
data: this.data,
error: "WebSocket error",
},
}));
}).bind(this);
}
close() {
this.ws.close();
}
}

46
apps/js-sdk/firecrawl/package-lock.json generated
View File

@ -1,16 +1,18 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "0.0.36", "version": "1.1.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "0.0.36", "version": "1.1.0",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"axios": "^1.6.8", "axios": "^1.6.8",
"dotenv": "^16.4.5", "dotenv": "^16.4.5",
"isows": "^1.0.4",
"typescript-event-target": "^1.1.1",
"uuid": "^9.0.1", "uuid": "^9.0.1",
"zod": "^3.23.8", "zod": "^3.23.8",
"zod-to-json-schema": "^3.23.0" "zod-to-json-schema": "^3.23.0"
@ -2137,6 +2139,20 @@
"integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
"dev": true "dev": true
}, },
"node_modules/isows": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/isows/-/isows-1.0.4.tgz",
"integrity": "sha512-hEzjY+x9u9hPmBom9IIAqdJCwNLax+xrPb51vEPpERoFlIxgmZcHzsT5jKG06nvInKOBGvReAVz80Umed5CczQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/wagmi-dev"
}
],
"peerDependencies": {
"ws": "*"
}
},
"node_modules/istanbul-lib-coverage": { "node_modules/istanbul-lib-coverage": {
"version": "3.2.2", "version": "3.2.2",
"resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz",
@ -3733,6 +3749,11 @@
"node": ">=14.17" "node": ">=14.17"
} }
}, },
"node_modules/typescript-event-target": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/typescript-event-target/-/typescript-event-target-1.1.1.tgz",
"integrity": "sha512-dFSOFBKV6uwaloBCCUhxlD3Pr/P1a/tJdcmPrTXCHlEFD3faj0mztjcGn6VBAhQ0/Bdy8K3VWrrqwbt/ffsYsg=="
},
"node_modules/undici-types": { "node_modules/undici-types": {
"version": "5.26.5", "version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
@ -3855,6 +3876,27 @@
"node": "^12.13.0 || ^14.15.0 || >=16.0.0" "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
} }
}, },
"node_modules/ws": {
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
"integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
"peer": true,
"engines": {
"node": ">=10.0.0"
},
"peerDependencies": {
"bufferutil": "^4.0.1",
"utf-8-validate": ">=5.0.2"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
}
},
"node_modules/y18n": { "node_modules/y18n": {
"version": "5.0.8", "version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

4
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "1.0.4", "version": "1.1.0",
"description": "JavaScript SDK for Firecrawl API", "description": "JavaScript SDK for Firecrawl API",
"main": "build/cjs/index.js", "main": "build/cjs/index.js",
"types": "types/index.d.ts", "types": "types/index.d.ts",
@ -30,6 +30,8 @@
"dependencies": { "dependencies": {
"axios": "^1.6.8", "axios": "^1.6.8",
"dotenv": "^16.4.5", "dotenv": "^16.4.5",
"isows": "^1.0.4",
"typescript-event-target": "^1.1.1",
"uuid": "^9.0.1", "uuid": "^9.0.1",
"zod": "^3.23.8", "zod": "^3.23.8",
"zod-to-json-schema": "^3.23.0" "zod-to-json-schema": "^3.23.0"

507
apps/js-sdk/firecrawl/src/index.ts
View File

@ -1,17 +1,17 @@
import axios, { AxiosResponse, AxiosRequestHeaders } from "axios"; import axios, { AxiosResponse, AxiosRequestHeaders } from "axios";
import { z } from "zod"; import { z } from "zod";
import { zodToJsonSchema } from "zod-to-json-schema"; import { zodToJsonSchema } from "zod-to-json-schema";
import { WebSocket } from "isows";
import { TypedEventTarget } from "typescript-event-target";
/** /**
* Configuration interface for FirecrawlApp. * Configuration interface for FirecrawlApp.
* @param apiKey - Optional API key for authentication. * @param apiKey - Optional API key for authentication.
* @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'. * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
* @param version - API version, either 'v0' or 'v1'.
*/ */
export interface FirecrawlAppConfig { export interface FirecrawlAppConfig {
apiKey?: string | null; apiKey?: string | null;
apiUrl?: string | null; apiUrl?: string | null;
version?: "v0" | "v1";
} }
/** /**
@ -54,17 +54,6 @@ export interface FirecrawlDocumentMetadata {
[key: string]: any; // Allows for additional metadata properties not explicitly defined. [key: string]: any; // Allows for additional metadata properties not explicitly defined.
} }
/**
* Metadata for a Firecrawl document on v0.
* Similar to FirecrawlDocumentMetadata but includes properties specific to API version v0.
*/
export interface FirecrawlDocumentMetadataV0 {
// Similar properties as FirecrawlDocumentMetadata with additional v0 specific adjustments
pageStatusCode?: number;
pageError?: string;
[key: string]: any;
}
/** /**
* Document interface for Firecrawl. * Document interface for Firecrawl.
* Represents a document retrieved or processed by Firecrawl. * Represents a document retrieved or processed by Firecrawl.
@ -77,28 +66,7 @@ export interface FirecrawlDocument {
links?: string[]; links?: string[];
extract?: Record<any, any>; extract?: Record<any, any>;
screenshot?: string; screenshot?: string;
metadata: FirecrawlDocumentMetadata; metadata?: FirecrawlDocumentMetadata;
}
/**
* Document interface for Firecrawl on v0.
* Represents a document specifically for API version v0 with additional properties.
*/
export interface FirecrawlDocumentV0 {
id?: string;
url?: string;
content: string;
markdown?: string;
html?: string;
llm_extraction?: Record<string, any>;
createdAt?: Date;
updatedAt?: Date;
type?: string;
metadata: FirecrawlDocumentMetadataV0;
childrenLinks?: string[];
provider?: string;
warning?: string;
index?: number;
} }
/** /**
@ -106,43 +74,17 @@ export interface FirecrawlDocumentV0 {
* Defines the options and configurations available for scraping web content. * Defines the options and configurations available for scraping web content.
*/ */
export interface ScrapeParams { export interface ScrapeParams {
formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract")[]; formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract" | "full@scrennshot")[];
headers?: Record<string, string>; headers?: Record<string, string>;
includeTags?: string[]; includeTags?: string[];
excludeTags?: string[]; excludeTags?: string[];
onlyMainContent?: boolean; onlyMainContent?: boolean;
screenshotMode?: "desktop" | "full-desktop" | "mobile" | "full-mobile";
waitFor?: number;
timeout?: number;
extract?: { extract?: {
prompt?: string; prompt?: string;
schema?: z.ZodSchema | any; schema?: z.ZodSchema | any;
systemPrompt?: string; systemPrompt?: string;
}; };
} waitFor?: number;
/**
* Parameters for scraping operations on v0.
* Includes page and extractor options specific to API version v0.
*/
export interface ScrapeParamsV0 {
pageOptions?: {
headers?: Record<string, string>;
includeHtml?: boolean;
includeRawHtml?: boolean;
onlyIncludeTags?: string[];
onlyMainContent?: boolean;
removeTags?: string[];
replaceAllPathsWithAbsolutePaths?: boolean;
screenshot?: boolean;
fullPageScreenshot?: boolean;
waitFor?: number;
};
extractorOptions?: {
mode?: "markdown" | "llm-extraction" | "llm-extraction-from-raw-html" | "llm-extraction-from-markdown";
extractionPrompt?: string;
extractionSchema?: Record<string, any> | z.ZodSchema | any;
};
timeout?: number; timeout?: number;
} }
@ -151,21 +93,11 @@ export interface ScrapeParamsV0 {
* Defines the structure of the response received after a scraping operation. * Defines the structure of the response received after a scraping operation.
*/ */
export interface ScrapeResponse extends FirecrawlDocument { export interface ScrapeResponse extends FirecrawlDocument {
success: boolean; success: true;
warning?: string; warning?: string;
error?: string; error?: string;
} }
/**
* Response interface for scraping operations on v0.
* Similar to ScrapeResponse but tailored for responses from API version v0.
*/
export interface ScrapeResponseV0 {
success: boolean;
data?: FirecrawlDocumentV0;
error?: string;
}
/** /**
* Parameters for crawling operations. * Parameters for crawling operations.
* Includes options for both scraping and mapping during a crawl. * Includes options for both scraping and mapping during a crawl.
@ -181,37 +113,6 @@ export interface CrawlParams {
scrapeOptions?: ScrapeParams; scrapeOptions?: ScrapeParams;
} }
/**
* Parameters for crawling operations on v0.
* Tailored for API version v0, includes specific options for crawling.
*/
export interface CrawlParamsV0 {
crawlerOptions?: {
includes?: string[];
excludes?: string[];
generateImgAltText?: boolean;
returnOnlyUrls?: boolean;
maxDepth?: number;
mode?: "default" | "fast";
ignoreSitemap?: boolean;
limit?: number;
allowBackwardCrawling?: boolean;
allowExternalContentLinks?: boolean;
};
pageOptions?: {
headers?: Record<string, string>;
includeHtml?: boolean;
includeRawHtml?: boolean;
onlyIncludeTags?: string[];
onlyMainContent?: boolean;
removeTags?: string[];
replaceAllPathsWithAbsolutePaths?: boolean;
screenshot?: boolean;
fullPageScreenshot?: boolean;
waitFor?: number;
};
}
/** /**
* Response interface for crawling operations. * Response interface for crawling operations.
* Defines the structure of the response received after initiating a crawl. * Defines the structure of the response received after initiating a crawl.
@ -219,17 +120,7 @@ export interface CrawlParamsV0 {
export interface CrawlResponse { export interface CrawlResponse {
id?: string; id?: string;
url?: string; url?: string;
success: boolean; success: true;
error?: string;
}
/**
* Response interface for crawling operations on v0.
* Similar to CrawlResponse but tailored for responses from API version v0.
*/
export interface CrawlResponseV0 {
jobId?: string;
success: boolean;
error?: string; error?: string;
} }
@ -238,7 +129,7 @@ export interface CrawlResponseV0 {
* Provides detailed status of a crawl job including progress and results. * Provides detailed status of a crawl job including progress and results.
*/ */
export interface CrawlStatusResponse { export interface CrawlStatusResponse {
success: boolean; success: true;
total: number; total: number;
completed: number; completed: number;
creditsUsed: number; creditsUsed: number;
@ -249,23 +140,6 @@ export interface CrawlStatusResponse {
error?: string; error?: string;
} }
/**
* Response interface for job status checks on v0.
* Tailored for API version v0, provides status and partial data of a crawl job.
*/
export interface CrawlStatusResponseV0 {
success: boolean;
status: string;
current?: number;
current_url?: string;
current_step?: string;
total?: number;
data?: FirecrawlDocumentV0[];
partial_data?: FirecrawlDocumentV0[];
error?: string;
}
/** /**
* Parameters for mapping operations. * Parameters for mapping operations.
* Defines options for mapping URLs during a crawl. * Defines options for mapping URLs during a crawl.
@ -282,57 +156,35 @@ export interface MapParams {
* Defines the structure of the response received after a mapping operation. * Defines the structure of the response received after a mapping operation.
*/ */
export interface MapResponse { export interface MapResponse {
success: boolean; success: true;
links?: string[]; links?: string[];
error?: string; error?: string;
} }
/** /**
* Parameters for searching operations on v0. * Error response interface.
* Tailored for API version v0, includes specific options for searching content. * Defines the structure of the response received when an error occurs.
*/ */
export interface SearchParamsV0 { export interface ErrorResponse {
pageOptions?: { success: false;
onlyMainContent?: boolean; error: string;
fetchPageContent?: boolean;
includeHtml?: boolean;
includeRawHtml?: boolean;
};
searchOptions?: {
limit?: number;
};
}
/**
* Response interface for searching operations on v0.
* Defines the structure of the response received after a search operation on v0.
*/
export interface SearchResponseV0 {
success: boolean;
data?: FirecrawlDocumentV0[];
error?: string;
} }
/** /**
* Main class for interacting with the Firecrawl API. * Main class for interacting with the Firecrawl API.
* Provides methods for scraping, searching, crawling, and mapping web content. * Provides methods for scraping, searching, crawling, and mapping web content.
*/ */
export default class FirecrawlApp<T extends "v0" | "v1"> { export default class FirecrawlApp {
private apiKey: string; public apiKey: string;
private apiUrl: string; public apiUrl: string;
public version: T;
/** /**
* Initializes a new instance of the FirecrawlApp class. * Initializes a new instance of the FirecrawlApp class.
* @param config - Configuration options for the FirecrawlApp instance. * @param config - Configuration options for the FirecrawlApp instance.
*/ */
constructor({ apiKey = null, apiUrl = null, version = "v1" }: FirecrawlAppConfig) { constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
this.apiKey = apiKey || ""; this.apiKey = apiKey || "";
this.apiUrl = apiUrl || "https://api.firecrawl.dev"; this.apiUrl = apiUrl || "https://api.firecrawl.dev";
this.version = version as T;
if (!this.apiKey) {
throw new Error("No API key provided");
}
} }
/** /**
@ -343,8 +195,8 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
*/ */
async scrapeUrl( async scrapeUrl(
url: string, url: string,
params?: ScrapeParams | ScrapeParamsV0 params?: ScrapeParams
): Promise<this['version'] extends 'v0' ? ScrapeResponseV0 : ScrapeResponse> { ): Promise<ScrapeResponse | ErrorResponse> {
const headers: AxiosRequestHeaders = { const headers: AxiosRequestHeaders = {
"Content-Type": "application/json", "Content-Type": "application/json",
Authorization: `Bearer ${this.apiKey}`, Authorization: `Bearer ${this.apiKey}`,
@ -383,19 +235,19 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
} }
try { try {
const response: AxiosResponse = await axios.post( const response: AxiosResponse = await axios.post(
this.apiUrl + `/${this.version}/scrape`, this.apiUrl + `/v1/scrape`,
jsonData, jsonData,
{ headers } { headers }
); );
if (response.status === 200) { if (response.status === 200) {
const responseData = response.data; const responseData = response.data;
if (responseData.success) { if (responseData.success) {
return (this.version === 'v0' ? responseData as ScrapeResponseV0 : { return {
success: true, success: true,
warning: responseData.warning, warning: responseData.warning,
error: responseData.error, error: responseData.error,
...responseData.data ...responseData.data
}) as ScrapeResponse; };
} else { } else {
throw new Error(`Failed to scrape URL. Error: ${responseData.error}`); throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
} }
@ -405,100 +257,47 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
} catch (error: any) { } catch (error: any) {
throw new Error(error.message); throw new Error(error.message);
} }
return { success: false, error: "Internal server error." } as this['version'] extends 'v0' ? ScrapeResponseV0 : ScrapeResponse; return { success: false, error: "Internal server error." };
} }
/** /**
* Searches for a query using the Firecrawl API. * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
* @param query - The query to search for. * @param query - The search query string.
* @param params - Additional parameters for the search request. * @param params - Additional parameters for the search.
* @returns The response from the search operation. * @returns Throws an error advising to use version 0 of the API.
*/ */
async search( async search(
query: string, query: string,
params?: SearchParamsV0 params?: any
): Promise<SearchResponseV0> { ): Promise<any> {
if (this.version === "v1") { throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
}
const headers: AxiosRequestHeaders = {
"Content-Type": "application/json",
Authorization: `Bearer ${this.apiKey}`,
} as AxiosRequestHeaders;
let jsonData: any = { query };
if (params) {
jsonData = { ...jsonData, ...params };
}
try {
const response: AxiosResponse = await axios.post(
this.apiUrl + "/v0/search",
jsonData,
{ headers }
);
if (response.status === 200) {
const responseData = response.data;
if (responseData.success) {
return responseData;
} else {
throw new Error(`Failed to search. Error: ${responseData.error}`);
}
} else {
this.handleError(response, "search");
}
} catch (error: any) {
throw new Error(error.message);
}
return { success: false, error: "Internal server error." };
} }
/** /**
* Initiates a crawl job for a URL using the Firecrawl API. * Initiates a crawl job for a URL using the Firecrawl API.
* @param url - The URL to crawl. * @param url - The URL to crawl.
* @param params - Additional parameters for the crawl request. * @param params - Additional parameters for the crawl request.
* @param waitUntilDone - Whether to wait for the crawl job to complete.
* @param pollInterval - Time in seconds for job status checks. * @param pollInterval - Time in seconds for job status checks.
* @param idempotencyKey - Optional idempotency key for the request. * @param idempotencyKey - Optional idempotency key for the request.
* @returns The response from the crawl operation. * @returns The response from the crawl operation.
*/ */
async crawlUrl( async crawlUrl(
url: string, url: string,
params?: this['version'] extends 'v0' ? CrawlParamsV0 : CrawlParams, params?: CrawlParams,
waitUntilDone: boolean = true,
pollInterval: number = 2, pollInterval: number = 2,
idempotencyKey?: string idempotencyKey?: string
): Promise< ): Promise<CrawlStatusResponse | ErrorResponse> {
this['version'] extends 'v0'
? CrawlResponseV0 | CrawlStatusResponseV0 | FirecrawlDocumentV0[]
: CrawlResponse | CrawlStatusResponse
> {
const headers = this.prepareHeaders(idempotencyKey); const headers = this.prepareHeaders(idempotencyKey);
let jsonData: any = { url, ...params }; let jsonData: any = { url, ...params };
try { try {
const response: AxiosResponse = await this.postRequest( const response: AxiosResponse = await this.postRequest(
this.apiUrl + `/${this.version}/crawl`, this.apiUrl + `/v1/crawl`,
jsonData, jsonData,
headers headers
); );
if (response.status === 200) { if (response.status === 200) {
const id: string = this.version === 'v0' ? response.data.jobId : response.data.id; const id: string = response.data.id;
let checkUrl: string | undefined = undefined; return this.monitorJobStatus(id, headers, pollInterval);
if (waitUntilDone) {
if (this.version === 'v1') { checkUrl = response.data.url }
return this.monitorJobStatus(id, headers, pollInterval, checkUrl);
} else {
if (this.version === 'v0') {
return {
success: true,
jobId: id
} as CrawlResponseV0;
} else {
return {
success: true,
id: id
} as CrawlResponse;
}
}
} else { } else {
this.handleError(response, "start crawl job"); this.handleError(response, "start crawl job");
} }
@ -509,7 +308,35 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
throw new Error(error.message); throw new Error(error.message);
} }
} }
return { success: false, error: "Internal server error." } as this['version'] extends 'v0' ? CrawlResponseV0 : CrawlResponse; return { success: false, error: "Internal server error." };
}
async asyncCrawlUrl(
url: string,
params?: CrawlParams,
idempotencyKey?: string
): Promise<CrawlResponse | ErrorResponse> {
const headers = this.prepareHeaders(idempotencyKey);
let jsonData: any = { url, ...params };
try {
const response: AxiosResponse = await this.postRequest(
this.apiUrl + `/v1/crawl`,
jsonData,
headers
);
if (response.status === 200) {
return response.data;
} else {
this.handleError(response, "start crawl job");
}
} catch (error: any) {
if (error.response?.data?.error) {
throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`);
} else {
throw new Error(error.message);
}
}
return { success: false, error: "Internal server error." };
} }
/** /**
@ -517,7 +344,7 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
* @param id - The ID of the crawl operation. * @param id - The ID of the crawl operation.
* @returns The response containing the job status. * @returns The response containing the job status.
*/ */
async checkCrawlStatus(id?: string): Promise<this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse> { async checkCrawlStatus(id?: string): Promise<CrawlStatusResponse | ErrorResponse> {
if (!id) { if (!id) {
throw new Error("No crawl ID provided"); throw new Error("No crawl ID provided");
} }
@ -525,71 +352,52 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
const headers: AxiosRequestHeaders = this.prepareHeaders(); const headers: AxiosRequestHeaders = this.prepareHeaders();
try { try {
const response: AxiosResponse = await this.getRequest( const response: AxiosResponse = await this.getRequest(
this.version === 'v1' ? `${this.apiUrl}/v1/crawl/${id}`,
`${this.apiUrl}/${this.version}/crawl/${id}` :
`${this.apiUrl}/${this.version}/crawl/status/${id}`,
headers headers
); );
if (response.status === 200) { if (response.status === 200) {
if (this.version === 'v0') { return ({
return ({ success: true,
success: true, status: response.data.status,
status: response.data.status, total: response.data.total,
current: response.data.current, completed: response.data.completed,
current_url: response.data.current_url, creditsUsed: response.data.creditsUsed,
current_step: response.data.current_step, expiresAt: new Date(response.data.expiresAt),
total: response.data.total, next: response.data.next,
data: response.data.data, data: response.data.data,
partial_data: !response.data.data error: response.data.error
? response.data.partial_data })
: undefined,
} as CrawlStatusResponseV0) as this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse;
} else {
return ({
success: true,
status: response.data.status,
total: response.data.total,
completed: response.data.completed,
creditsUsed: response.data.creditsUsed,
expiresAt: new Date(response.data.expiresAt),
next: response.data.next,
data: response.data.data,
error: response.data.error
} as CrawlStatusResponse) as this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse;
}
} else { } else {
this.handleError(response, "check crawl status"); this.handleError(response, "check crawl status");
} }
} catch (error: any) { } catch (error: any) {
throw new Error(error.message); throw new Error(error.message);
} }
return { success: false, error: "Internal server error." };
return this.version === 'v0' ?
({
success: false,
status: "unknown",
current: 0,
current_url: "",
current_step: "",
total: 0,
error: "Internal server error.",
} as this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse) :
({
success: false,
error: "Internal server error.",
} as this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse);
} }
async mapUrl(url: string, params?: MapParams): Promise<MapResponse> { async crawlUrlAndWatch(
if (this.version == 'v0') { url: string,
throw new Error("Map is not supported in v0"); params?: CrawlParams,
idempotencyKey?: string,
) {
const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
if (crawl.success && crawl.id) {
const id = crawl.id;
return new CrawlWatcher(id, this);
} }
throw new Error("Crawl job failed to start");
}
async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
const headers = this.prepareHeaders(); const headers = this.prepareHeaders();
let jsonData: { url: string } & MapParams = { url, ...params }; let jsonData: { url: string } & MapParams = { url, ...params };
try { try {
const response: AxiosResponse = await this.postRequest( const response: AxiosResponse = await this.postRequest(
this.apiUrl + `/${this.version}/map`, this.apiUrl + `/v1/map`,
jsonData, jsonData,
headers headers
); );
@ -601,7 +409,7 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
} catch (error: any) { } catch (error: any) {
throw new Error(error.message); throw new Error(error.message);
} }
return { success: false, error: "Internal server error." } as MapResponse; return { success: false, error: "Internal server error." };
} }
/** /**
@ -656,25 +464,18 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
async monitorJobStatus( async monitorJobStatus(
id: string, id: string,
headers: AxiosRequestHeaders, headers: AxiosRequestHeaders,
checkInterval: number, checkInterval: number
checkUrl?: string ): Promise<CrawlStatusResponse> {
): Promise<this['version'] extends 'v0' ? CrawlStatusResponseV0 | FirecrawlDocumentV0[] : CrawlStatusResponse> {
let apiUrl: string = '';
while (true) { while (true) {
if (this.version === 'v1') {
apiUrl = checkUrl ?? `${this.apiUrl}/v1/crawl/${id}`;
} else if (this.version === 'v0') {
apiUrl = `${this.apiUrl}/v0/crawl/status/${id}`;
}
const statusResponse: AxiosResponse = await this.getRequest( const statusResponse: AxiosResponse = await this.getRequest(
apiUrl, `${this.apiUrl}/v1/crawl/${id}`,
headers headers
); );
if (statusResponse.status === 200) { if (statusResponse.status === 200) {
const statusData = statusResponse.data; const statusData = statusResponse.data;
if (statusData.status === "completed") { if (statusData.status === "completed") {
if ("data" in statusData) { if ("data" in statusData) {
return this.version === 'v0' ? statusData.data : statusData; return statusData;
} else { } else {
throw new Error("Crawl job completed but no data was returned"); throw new Error("Crawl job completed but no data was returned");
} }
@ -715,3 +516,111 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
} }
} }
} }
interface CrawlWatcherEvents {
document: CustomEvent<FirecrawlDocument>,
done: CustomEvent<{
status: CrawlStatusResponse["status"];
data: FirecrawlDocument[];
}>,
error: CustomEvent<{
status: CrawlStatusResponse["status"],
data: FirecrawlDocument[],
error: string,
}>,
}
export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
private ws: WebSocket;
public data: FirecrawlDocument[];
public status: CrawlStatusResponse["status"];
constructor(id: string, app: FirecrawlApp) {
super();
this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
this.status = "scraping";
this.data = [];
type ErrorMessage = {
type: "error",
error: string,
}
type CatchupMessage = {
type: "catchup",
data: CrawlStatusResponse,
}
type DocumentMessage = {
type: "document",
data: FirecrawlDocument,
}
type DoneMessage = { type: "done" }
type Message = ErrorMessage | CatchupMessage | DoneMessage | DocumentMessage;
const messageHandler = (msg: Message) => {
if (msg.type === "done") {
this.status = "completed";
this.dispatchTypedEvent("done", new CustomEvent("done", {
detail: {
status: this.status,
data: this.data,
},
}));
} else if (msg.type === "error") {
this.status = "failed";
this.dispatchTypedEvent("error", new CustomEvent("error", {
detail: {
status: this.status,
data: this.data,
error: msg.error,
},
}));
} else if (msg.type === "catchup") {
this.status = msg.data.status;
this.data.push(...(msg.data.data ?? []));
for (const doc of this.data) {
this.dispatchTypedEvent("document", new CustomEvent("document", {
detail: doc,
}));
}
} else if (msg.type === "document") {
this.dispatchTypedEvent("document", new CustomEvent("document", {
detail: msg.data,
}));
}
}
this.ws.onmessage = ((ev: MessageEvent) => {
if (typeof ev.data !== "string") {
this.ws.close();
return;
}
const msg = JSON.parse(ev.data) as Message;
messageHandler(msg);
}).bind(this);
this.ws.onclose = ((ev: CloseEvent) => {
const msg = JSON.parse(ev.reason) as Message;
messageHandler(msg);
}).bind(this);
this.ws.onerror = ((_: Event) => {
this.status = "failed"
this.dispatchTypedEvent("error", new CustomEvent("error", {
detail: {
status: this.status,
data: this.data,
error: "WebSocket error",
},
}));
}).bind(this);
}
close() {
this.ws.close();
}
}

211
apps/js-sdk/firecrawl/types/index.d.ts vendored
View File

@ -1,15 +1,13 @@
import { AxiosResponse, AxiosRequestHeaders } from "axios"; import { AxiosResponse, AxiosRequestHeaders } from "axios";
import { z } from "zod"; import { TypedEventTarget } from "typescript-event-target";
/** /**
* Configuration interface for FirecrawlApp. * Configuration interface for FirecrawlApp.
* @param apiKey - Optional API key for authentication. * @param apiKey - Optional API key for authentication.
* @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'. * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
* @param version - API version, either 'v0' or 'v1'.
*/ */
export interface FirecrawlAppConfig { export interface FirecrawlAppConfig {
apiKey?: string | null; apiKey?: string | null;
apiUrl?: string | null; apiUrl?: string | null;
version?: "v0" | "v1";
} }
/** /**
* Metadata for a Firecrawl document. * Metadata for a Firecrawl document.
@ -50,15 +48,6 @@ export interface FirecrawlDocumentMetadata {
error?: string; error?: string;
[key: string]: any; [key: string]: any;
} }
/**
* Metadata for a Firecrawl document on v0.
* Similar to FirecrawlDocumentMetadata but includes properties specific to API version v0.
*/
export interface FirecrawlDocumentMetadataV0 {
pageStatusCode?: number;
pageError?: string;
[key: string]: any;
}
/** /**
* Document interface for Firecrawl. * Document interface for Firecrawl.
* Represents a document retrieved or processed by Firecrawl. * Represents a document retrieved or processed by Firecrawl.
@ -71,39 +60,18 @@ export interface FirecrawlDocument {
links?: string[]; links?: string[];
extract?: Record<any, any>; extract?: Record<any, any>;
screenshot?: string; screenshot?: string;
metadata: FirecrawlDocumentMetadata; metadata?: FirecrawlDocumentMetadata;
}
/**
* Document interface for Firecrawl on v0.
* Represents a document specifically for API version v0 with additional properties.
*/
export interface FirecrawlDocumentV0 {
id?: string;
url?: string;
content: string;
markdown?: string;
html?: string;
llm_extraction?: Record<string, any>;
createdAt?: Date;
updatedAt?: Date;
type?: string;
metadata: FirecrawlDocumentMetadataV0;
childrenLinks?: string[];
provider?: string;
warning?: string;
index?: number;
} }
/** /**
* Parameters for scraping operations. * Parameters for scraping operations.
* Defines the options and configurations available for scraping web content. * Defines the options and configurations available for scraping web content.
*/ */
export interface ScrapeParams { export interface ScrapeParams {
formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract")[]; formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract" | "full@scrennshot")[];
headers?: Record<string, string>; headers?: Record<string, string>;
includeTags?: string[]; includeTags?: string[];
excludeTags?: string[]; excludeTags?: string[];
onlyMainContent?: boolean; onlyMainContent?: boolean;
screenshotMode?: "desktop" | "full-desktop" | "mobile" | "full-mobile";
waitFor?: number; waitFor?: number;
timeout?: number; timeout?: number;
extract?: { extract?: {
@ -112,48 +80,15 @@ export interface ScrapeParams {
systemPrompt?: string; systemPrompt?: string;
}; };
} }
/**
* Parameters for scraping operations on v0.
* Includes page and extractor options specific to API version v0.
*/
export interface ScrapeParamsV0 {
pageOptions?: {
headers?: Record<string, string>;
includeHtml?: boolean;
includeRawHtml?: boolean;
onlyIncludeTags?: string[];
onlyMainContent?: boolean;
removeTags?: string[];
replaceAllPathsWithAbsolutePaths?: boolean;
screenshot?: boolean;
fullPageScreenshot?: boolean;
waitFor?: number;
};
extractorOptions?: {
mode?: "markdown" | "llm-extraction" | "llm-extraction-from-raw-html" | "llm-extraction-from-markdown";
extractionPrompt?: string;
extractionSchema?: Record<string, any> | z.ZodSchema | any;
};
timeout?: number;
}
/** /**
* Response interface for scraping operations. * Response interface for scraping operations.
* Defines the structure of the response received after a scraping operation. * Defines the structure of the response received after a scraping operation.
*/ */
export interface ScrapeResponse extends FirecrawlDocument { export interface ScrapeResponse extends FirecrawlDocument {
success: boolean; success: true;
warning?: string; warning?: string;
error?: string; error?: string;
} }
/**
* Response interface for scraping operations on v0.
* Similar to ScrapeResponse but tailored for responses from API version v0.
*/
export interface ScrapeResponseV0 {
success: boolean;
data?: FirecrawlDocumentV0;
error?: string;
}
/** /**
* Parameters for crawling operations. * Parameters for crawling operations.
* Includes options for both scraping and mapping during a crawl. * Includes options for both scraping and mapping during a crawl.
@ -168,36 +103,6 @@ export interface CrawlParams {
ignoreSitemap?: boolean; ignoreSitemap?: boolean;
scrapeOptions?: ScrapeParams; scrapeOptions?: ScrapeParams;
} }
/**
* Parameters for crawling operations on v0.
* Tailored for API version v0, includes specific options for crawling.
*/
export interface CrawlParamsV0 {
crawlerOptions?: {
includes?: string[];
excludes?: string[];
generateImgAltText?: boolean;
returnOnlyUrls?: boolean;
maxDepth?: number;
mode?: "default" | "fast";
ignoreSitemap?: boolean;
limit?: number;
allowBackwardCrawling?: boolean;
allowExternalContentLinks?: boolean;
};
pageOptions?: {
headers?: Record<string, string>;
includeHtml?: boolean;
includeRawHtml?: boolean;
onlyIncludeTags?: string[];
onlyMainContent?: boolean;
removeTags?: string[];
replaceAllPathsWithAbsolutePaths?: boolean;
screenshot?: boolean;
fullPageScreenshot?: boolean;
waitFor?: number;
};
}
/** /**
* Response interface for crawling operations. * Response interface for crawling operations.
* Defines the structure of the response received after initiating a crawl. * Defines the structure of the response received after initiating a crawl.
@ -205,16 +110,7 @@ export interface CrawlParamsV0 {
export interface CrawlResponse { export interface CrawlResponse {
id?: string; id?: string;
url?: string; url?: string;
success: boolean; success: true;
error?: string;
}
/**
* Response interface for crawling operations on v0.
* Similar to CrawlResponse but tailored for responses from API version v0.
*/
export interface CrawlResponseV0 {
jobId?: string;
success: boolean;
error?: string; error?: string;
} }
/** /**
@ -222,7 +118,7 @@ export interface CrawlResponseV0 {
* Provides detailed status of a crawl job including progress and results. * Provides detailed status of a crawl job including progress and results.
*/ */
export interface CrawlStatusResponse { export interface CrawlStatusResponse {
success: boolean; success: true;
total: number; total: number;
completed: number; completed: number;
creditsUsed: number; creditsUsed: number;
@ -232,21 +128,6 @@ export interface CrawlStatusResponse {
data?: FirecrawlDocument[]; data?: FirecrawlDocument[];
error?: string; error?: string;
} }
/**
* Response interface for job status checks on v0.
* Tailored for API version v0, provides status and partial data of a crawl job.
*/
export interface CrawlStatusResponseV0 {
success: boolean;
status: string;
current?: number;
current_url?: string;
current_step?: string;
total?: number;
data?: FirecrawlDocumentV0[];
partial_data?: FirecrawlDocumentV0[];
error?: string;
}
/** /**
* Parameters for mapping operations. * Parameters for mapping operations.
* Defines options for mapping URLs during a crawl. * Defines options for mapping URLs during a crawl.
@ -262,78 +143,62 @@ export interface MapParams {
* Defines the structure of the response received after a mapping operation. * Defines the structure of the response received after a mapping operation.
*/ */
export interface MapResponse { export interface MapResponse {
success: boolean; success: true;
links?: string[]; links?: string[];
error?: string; error?: string;
} }
/** /**
* Parameters for searching operations on v0. * Error response interface.
* Tailored for API version v0, includes specific options for searching content. * Defines the structure of the response received when an error occurs.
*/ */
export interface SearchParamsV0 { export interface ErrorResponse {
pageOptions?: { success: false;
onlyMainContent?: boolean; error: string;
fetchPageContent?: boolean;
includeHtml?: boolean;
includeRawHtml?: boolean;
};
searchOptions?: {
limit?: number;
};
}
/**
* Response interface for searching operations on v0.
* Defines the structure of the response received after a search operation on v0.
*/
export interface SearchResponseV0 {
success: boolean;
data?: FirecrawlDocumentV0[];
error?: string;
} }
/** /**
* Main class for interacting with the Firecrawl API. * Main class for interacting with the Firecrawl API.
* Provides methods for scraping, searching, crawling, and mapping web content. * Provides methods for scraping, searching, crawling, and mapping web content.
*/ */
export default class FirecrawlApp<T extends "v0" | "v1"> { export default class FirecrawlApp {
private apiKey; apiKey: string;
private apiUrl; apiUrl: string;
version: T;
/** /**
* Initializes a new instance of the FirecrawlApp class. * Initializes a new instance of the FirecrawlApp class.
* @param config - Configuration options for the FirecrawlApp instance. * @param config - Configuration options for the FirecrawlApp instance.
*/ */
constructor({ apiKey, apiUrl, version }: FirecrawlAppConfig); constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
/** /**
* Scrapes a URL using the Firecrawl API. * Scrapes a URL using the Firecrawl API.
* @param url - The URL to scrape. * @param url - The URL to scrape.
* @param params - Additional parameters for the scrape request. * @param params - Additional parameters for the scrape request.
* @returns The response from the scrape operation. * @returns The response from the scrape operation.
*/ */
scrapeUrl(url: string, params?: ScrapeParams | ScrapeParamsV0): Promise<this['version'] extends 'v0' ? ScrapeResponseV0 : ScrapeResponse>; scrapeUrl(url: string, params?: ScrapeParams): Promise<ScrapeResponse | ErrorResponse>;
/** /**
* Searches for a query using the Firecrawl API. * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
* @param query - The query to search for. * @param query - The search query string.
* @param params - Additional parameters for the search request. * @param params - Additional parameters for the search.
* @returns The response from the search operation. * @returns Throws an error advising to use version 0 of the API.
*/ */
search(query: string, params?: SearchParamsV0): Promise<SearchResponseV0>; search(query: string, params?: any): Promise<any>;
/** /**
* Initiates a crawl job for a URL using the Firecrawl API. * Initiates a crawl job for a URL using the Firecrawl API.
* @param url - The URL to crawl. * @param url - The URL to crawl.
* @param params - Additional parameters for the crawl request. * @param params - Additional parameters for the crawl request.
* @param waitUntilDone - Whether to wait for the crawl job to complete.
* @param pollInterval - Time in seconds for job status checks. * @param pollInterval - Time in seconds for job status checks.
* @param idempotencyKey - Optional idempotency key for the request. * @param idempotencyKey - Optional idempotency key for the request.
* @returns The response from the crawl operation. * @returns The response from the crawl operation.
*/ */
crawlUrl(url: string, params?: this['version'] extends 'v0' ? CrawlParamsV0 : CrawlParams, waitUntilDone?: boolean, pollInterval?: number, idempotencyKey?: string): Promise<this['version'] extends 'v0' ? CrawlResponseV0 | CrawlStatusResponseV0 | FirecrawlDocumentV0[] : CrawlResponse | CrawlStatusResponse>; crawlUrl(url: string, params?: CrawlParams, pollInterval?: number, idempotencyKey?: string): Promise<CrawlStatusResponse | ErrorResponse>;
asyncCrawlUrl(url: string, params?: CrawlParams, idempotencyKey?: string): Promise<CrawlResponse | ErrorResponse>;
/** /**
* Checks the status of a crawl job using the Firecrawl API. * Checks the status of a crawl job using the Firecrawl API.
* @param id - The ID of the crawl operation. * @param id - The ID of the crawl operation.
* @returns The response containing the job status. * @returns The response containing the job status.
*/ */
checkCrawlStatus(id?: string): Promise<this['version'] extends 'v0' ? CrawlStatusResponseV0 : CrawlStatusResponse>; checkCrawlStatus(id?: string): Promise<CrawlStatusResponse | ErrorResponse>;
mapUrl(url: string, params?: MapParams): Promise<MapResponse>; crawlUrlAndWatch(url: string, params?: CrawlParams, idempotencyKey?: string): Promise<CrawlWatcher>;
mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse>;
/** /**
* Prepares the headers for an API request. * Prepares the headers for an API request.
* @param idempotencyKey - Optional key to ensure idempotency. * @param idempotencyKey - Optional key to ensure idempotency.
@ -363,7 +228,7 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
* @param checkUrl - Optional URL to check the status (used for v1 API) * @param checkUrl - Optional URL to check the status (used for v1 API)
* @returns The final job status or data. * @returns The final job status or data.
*/ */
monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number, checkUrl?: string): Promise<this['version'] extends 'v0' ? CrawlStatusResponseV0 | FirecrawlDocumentV0[] : CrawlStatusResponse>; monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse>;
/** /**
* Handles errors from API responses. * Handles errors from API responses.
* @param {AxiosResponse} response - The response from the API. * @param {AxiosResponse} response - The response from the API.
@ -371,3 +236,23 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
*/ */
handleError(response: AxiosResponse, action: string): void; handleError(response: AxiosResponse, action: string): void;
} }
interface CrawlWatcherEvents {
document: CustomEvent<FirecrawlDocument>;
done: CustomEvent<{
status: CrawlStatusResponse["status"];
data: FirecrawlDocument[];
}>;
error: CustomEvent<{
status: CrawlStatusResponse["status"];
data: FirecrawlDocument[];
error: string;
}>;
}
export declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
private ws;
data: FirecrawlDocument[];
status: CrawlStatusResponse["status"];
constructor(id: string, app: FirecrawlApp);
close(): void;
}
export {};

28
apps/js-sdk/test.ts
View File

@ -1,28 +0,0 @@
import FirecrawlApp from "@mendable/firecrawl-js";
import { z } from "zod";
async function a() {
const app = new FirecrawlApp({
apiKey: "fc-YOUR_API_KEY",
});
// Define schema to extract contents into
const schema = z.object({
top: z
.array(
z.object({
title: z.string(),
points: z.number(),
by: z.string(),
commentsURL: z.string(),
})
)
.length(5)
.describe("Top 5 stories on Hacker News"),
});
const scrapeResult = await app.scrapeUrl("https://firecrawl.dev", {
extractorOptions: { extractionSchema: schema },
});
console.log(scrapeResult.data["llm_extraction"]);
}
a();

59
apps/python-sdk/README.md
View File

@ -81,22 +81,20 @@ print(data["llm_extraction"])
To crawl a website, use the `crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format. To crawl a website, use the `crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
The `wait_until_done` parameter determines whether the method should wait for the crawl job to complete before returning the result. If set to `True`, the method will periodically check the status of the crawl job until it is completed or the specified `timeout` (in seconds) is reached. If set to `False`, the method will return immediately with the job ID, and you can manually check the status of the crawl job using the `check_crawl_status` method.
```python ```python
crawl_status = app.crawl_url( idempotency_key = str(uuid.uuid4()) # optional idempotency key
'https://firecrawl.dev', crawl_result = app.crawl_url('firecrawl.dev', {'excludePaths': ['blog/*']}, 2, idempotency_key)
params={ print(crawl_result)
'limit': 100,
'scrapeOptions': {'formats': ['markdown', 'html']}
},
wait_until_done=True,
poll_interval=30
)
print(crawl_status)
``` ```
If `wait_until_done` is set to `True`, the `crawl_url` method will return the crawl result once the job is completed. If the job fails or is stopped, an exception will be raised. ### Asynchronous Crawl a Website
To crawl a website asynchronously, use the `async_crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
```python
crawl_result = app.async_crawl_url('firecrawl.dev', {'excludePaths': ['blog/*']}, "")
print(crawl_result)
```
### Checking Crawl Status ### Checking Crawl Status
@ -117,6 +115,41 @@ map_result = app.map_url('https://example.com')
print(map_result) print(map_result)
``` ```
### Crawl a website with WebSockets
To crawl a website with WebSockets, use the `crawl_url_and_watch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
```python
# inside an async function...
nest_asyncio.apply()
# Define event handlers
def on_document(detail):
print("DOC", detail)
def on_error(detail):
print("ERR", detail['error'])
def on_done(detail):
print("DONE", detail['status'])
# Function to start the crawl and watch process
async def start_crawl_and_watch():
# Initiate the crawl job and get the watcher
watcher = app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })
# Add event listeners
watcher.add_event_listener("document", on_document)
watcher.add_event_listener("error", on_error)
watcher.add_event_listener("done", on_done)
# Start the watcher
await watcher.connect()
# Run the event loop
await start_crawl_and_watch()
```
## Error Handling ## Error Handling
The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message.

54
apps/python-sdk/example.py
View File

@ -1,3 +1,5 @@
import time
import nest_asyncio
import uuid import uuid
from firecrawl.firecrawl import FirecrawlApp from firecrawl.firecrawl import FirecrawlApp
@ -8,10 +10,26 @@ scrape_result = app.scrape_url('firecrawl.dev')
print(scrape_result['markdown']) print(scrape_result['markdown'])
# Crawl a website: # Crawl a website:
crawl_result = app.crawl_url('docs.firecrawl.dev', {}, True, 2) idempotency_key = str(uuid.uuid4()) # optional idempotency key
crawl_result = app.crawl_url('firecrawl.dev', {'excludePaths': ['blog/*']}, 2, idempotency_key)
print(crawl_result) print(crawl_result)
# Asynchronous Crawl a website:
async_result = app.async_crawl_url('firecrawl.dev', {'excludePaths': ['blog/*']}, "")
print(async_result)
crawl_status = app.check_crawl_status(async_result['id'])
print(crawl_status)
attempts = 15
while attempts > 0 and crawl_status['status'] != 'completed':
print(crawl_status)
crawl_status = app.check_crawl_status(async_result['id'])
attempts -= 1
time.sleep(1)
crawl_status = app.get_crawl_status(async_result['id'])
print(crawl_status)
# LLM Extraction: # LLM Extraction:
# Define schema to extract contents into using pydantic # Define schema to extract contents into using pydantic
@ -73,4 +91,36 @@ llm_extraction_result = app2.scrape_url('https://news.ycombinator.com', {
} }
}) })
print(llm_extraction_result['llm_extraction']) # print(llm_extraction_result['llm_extraction'])
# Map a website:
map_result = app.map_url('https://firecrawl.dev', { 'search': 'blog' })
print(map_result)
# Crawl a website with WebSockets:
# inside an async function...
nest_asyncio.apply()
# Define event handlers
def on_document(detail):
print("DOC", detail)
def on_error(detail):
print("ERR", detail['error'])
def on_done(detail):
print("DONE", detail['status'])
# Function to start the crawl and watch process
async def start_crawl_and_watch():
# Initiate the crawl job and get the watcher
watcher = app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })
# Add event listeners
watcher.add_event_listener("document", on_document)
watcher.add_event_listener("error", on_error)
watcher.add_event_listener("done", on_done)
# Start the watcher
await watcher.connect()

75
apps/python-sdk/examplev0.py
View File

@ -1,75 +0,0 @@
import uuid
from firecrawl.firecrawl import FirecrawlApp
app = FirecrawlApp(api_key="fc-YOUR_API_KEY")
# Scrape a website:
scrape_result = app.scrape_url('firecrawl.dev')
print(scrape_result['markdown'])
# Crawl a website:
idempotency_key = str(uuid.uuid4()) # optional idempotency key
crawl_result = app.crawl_url('mendable.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, idempotency_key)
print(crawl_result)
# LLM Extraction:
# Define schema to extract contents into using pydantic
from pydantic import BaseModel, Field
from typing import List
class ArticleSchema(BaseModel):
title: str
points: int
by: str
commentsURL: str
class TopArticlesSchema(BaseModel):
top: List[ArticleSchema] = Field(..., max_items=5, description="Top 5 stories")
llm_extraction_result = app.scrape_url('https://news.ycombinator.com', {
'extractorOptions': {
'extractionSchema': TopArticlesSchema.model_json_schema(),
'mode': 'llm-extraction'
},
'pageOptions':{
'onlyMainContent': True
}
})
print(llm_extraction_result['llm_extraction'])
# Define schema to extract contents into using json schema
json_schema = {
"type": "object",
"properties": {
"top": {
"type": "array",
"items": {
"type": "object",
"properties": {
"title": {"type": "string"},
"points": {"type": "number"},
"by": {"type": "string"},
"commentsURL": {"type": "string"}
},
"required": ["title", "points", "by", "commentsURL"]
},
"minItems": 5,
"maxItems": 5,
"description": "Top 5 stories on Hacker News"
}
},
"required": ["top"]
}
llm_extraction_result = app.scrape_url('https://news.ycombinator.com', {
'extractorOptions': {
'extractionSchema': json_schema,
'mode': 'llm-extraction'
},
'pageOptions':{
'onlyMainContent': True
}
})
print(llm_extraction_result['llm_extraction'])

2
apps/python-sdk/firecrawl/__init__.py
View File

@ -13,7 +13,7 @@ import os
from .firecrawl import FirecrawlApp from .firecrawl import FirecrawlApp
__version__ = "1.0.1" __version__ = "1.1.1"
# Define the logger for the Firecrawl project # Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl") logger: logging.Logger = logging.getLogger("firecrawl")

225
apps/python-sdk/firecrawl/firecrawl.py
View File

@ -12,29 +12,30 @@ Classes:
import logging import logging
import os import os
import time import time
from typing import Any, Dict, Optional from typing import Any, Dict, Optional, List
import asyncio
import json
import requests import requests
import websockets
logger : logging.Logger = logging.getLogger("firecrawl") logger : logging.Logger = logging.getLogger("firecrawl")
class FirecrawlApp: class FirecrawlApp:
def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None, version: str = 'v1') -> None: def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> None:
""" """
Initialize the FirecrawlApp instance with API key, API URL, and version. Initialize the FirecrawlApp instance with API key, API URL.
Args: Args:
api_key (Optional[str]): API key for authenticating with the Firecrawl API. api_key (Optional[str]): API key for authenticating with the Firecrawl API.
api_url (Optional[str]): Base URL for the Firecrawl API. api_url (Optional[str]): Base URL for the Firecrawl API.
version (str): API version, either 'v0' or 'v1'.
""" """
self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY') self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY')
self.api_url = api_url or os.getenv('FIRECRAWL_API_URL', 'https://api.firecrawl.dev') self.api_url = api_url or os.getenv('FIRECRAWL_API_URL', 'https://api.firecrawl.dev')
self.version = version
if self.api_key is None: if self.api_key is None:
logger.warning("No API key provided") logger.warning("No API key provided")
raise ValueError('No API key provided') raise ValueError('No API key provided')
logger.debug(f"Initialized FirecrawlApp with API key: {self.api_key} and version: {self.version}") logger.debug(f"Initialized FirecrawlApp with API key: {self.api_key}")
def scrape_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any: def scrape_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any:
""" """
@ -84,7 +85,7 @@ class FirecrawlApp:
if key not in ['extract']: if key not in ['extract']:
scrape_params[key] = value scrape_params[key] = value
endpoint = f'/{self.version}/scrape' endpoint = f'/v1/scrape'
# Make the POST request with the prepared headers and JSON data # Make the POST request with the prepared headers and JSON data
response = requests.post( response = requests.post(
f'{self.api_url}{endpoint}', f'{self.api_url}{endpoint}',
@ -112,35 +113,14 @@ class FirecrawlApp:
Any: The search results if the request is successful. Any: The search results if the request is successful.
Raises: Raises:
NotImplementedError: If the search request is attempted on API version v1.
Exception: If the search request fails. Exception: If the search request fails.
""" """
if self.version == 'v1': raise NotImplementedError("Search is not supported in v1.")
raise NotImplementedError("Search is not supported in v1")
headers = self._prepare_headers()
json_data = {'query': query}
if params:
json_data.update(params)
response = requests.post(
f'{self.api_url}/v0/search',
headers=headers,
json=json_data
)
if response.status_code == 200:
response = response.json()
if response['success'] and 'data' in response:
return response['data']
else:
raise Exception(f'Failed to search. Error: {response["error"]}')
else:
self._handle_error(response, 'search')
def crawl_url(self, url: str, def crawl_url(self, url: str,
params: Optional[Dict[str, Any]] = None, params: Optional[Dict[str, Any]] = None,
wait_until_done: bool = True, poll_interval: Optional[int] = 2,
poll_interval: int = 2,
idempotency_key: Optional[str] = None) -> Any: idempotency_key: Optional[str] = None) -> Any:
""" """
Initiate a crawl job for the specified URL using the Firecrawl API. Initiate a crawl job for the specified URL using the Firecrawl API.
@ -148,8 +128,7 @@ class FirecrawlApp:
Args: Args:
url (str): The URL to crawl. url (str): The URL to crawl.
params (Optional[Dict[str, Any]]): Additional parameters for the crawl request. params (Optional[Dict[str, Any]]): Additional parameters for the crawl request.
wait_until_done (bool): Whether to wait until the crawl job is completed. poll_interval (Optional[int]): Time in seconds between status checks when waiting for job completion. Defaults to 2 seconds.
poll_interval (int): Time in seconds between status checks when waiting for job completion.
idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests. idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests.
Returns: Returns:
@ -158,28 +137,40 @@ class FirecrawlApp:
Raises: Raises:
Exception: If the crawl job initiation or monitoring fails. Exception: If the crawl job initiation or monitoring fails.
""" """
endpoint = f'/{self.version}/crawl' endpoint = f'/v1/crawl'
headers = self._prepare_headers(idempotency_key) headers = self._prepare_headers(idempotency_key)
json_data = {'url': url} json_data = {'url': url}
if params: if params:
json_data.update(params) json_data.update(params)
response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers) response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers)
if response.status_code == 200: if response.status_code == 200:
if self.version == 'v0': id = response.json().get('id')
id = response.json().get('jobId') return self._monitor_job_status(id, headers, poll_interval)
else:
id = response.json().get('id')
if wait_until_done: else:
check_url = None self._handle_error(response, 'start crawl job')
if self.version == 'v1':
check_url = response.json().get('url')
return self._monitor_job_status(id, headers, poll_interval, check_url) def async_crawl_url(self, url: str, params: Optional[Dict[str, Any]] = None, idempotency_key: Optional[str] = None) -> Dict[str, Any]:
else: """
if self.version == 'v0': Initiate a crawl job asynchronously.
return {'jobId': id}
else: Args:
return {'id': id} url (str): The URL to crawl.
params (Optional[Dict[str, Any]]): Additional parameters for the crawl request.
idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests.
Returns:
Dict[str, Any]: The response from the crawl initiation request.
"""
endpoint = f'/v1/crawl'
headers = self._prepare_headers(idempotency_key)
json_data = {'url': url}
if params:
json_data.update(params)
response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers)
if response.status_code == 200:
return response.json()
else: else:
self._handle_error(response, 'start crawl job') self._handle_error(response, 'start crawl job')
@ -196,50 +187,56 @@ class FirecrawlApp:
Raises: Raises:
Exception: If the status check request fails. Exception: If the status check request fails.
""" """
endpoint = f'/v1/crawl/{id}'
if self.version == 'v0':
endpoint = f'/{self.version}/crawl/status/{id}'
else:
endpoint = f'/{self.version}/crawl/{id}'
headers = self._prepare_headers() headers = self._prepare_headers()
response = self._get_request(f'{self.api_url}{endpoint}', headers) response = self._get_request(f'{self.api_url}{endpoint}', headers)
if response.status_code == 200: if response.status_code == 200:
data = response.json() data = response.json()
if self.version == 'v0': return {
return { 'success': True,
'success': True, 'status': data.get('status'),
'status': data.get('status'), 'total': data.get('total'),
'current': data.get('current'), 'completed': data.get('completed'),
'current_url': data.get('current_url'), 'creditsUsed': data.get('creditsUsed'),
'current_step': data.get('current_step'), 'expiresAt': data.get('expiresAt'),
'total': data.get('total'), 'next': data.get('next'),
'data': data.get('data'), 'data': data.get('data'),
'partial_data': data.get('partial_data') if not data.get('data') else None, 'error': data.get('error')
} }
elif self.version == 'v1':
return {
'success': True,
'status': data.get('status'),
'total': data.get('total'),
'completed': data.get('completed'),
'creditsUsed': data.get('creditsUsed'),
'expiresAt': data.get('expiresAt'),
'next': data.get('next'),
'data': data.get('data'),
'error': data.get('error')
}
else: else:
self._handle_error(response, 'check crawl status') self._handle_error(response, 'check crawl status')
def crawl_url_and_watch(self, url: str, params: Optional[Dict[str, Any]] = None, idempotency_key: Optional[str] = None) -> 'CrawlWatcher':
"""
Initiate a crawl job and return a CrawlWatcher to monitor the job via WebSocket.
Args:
url (str): The URL to crawl.
params (Optional[Dict[str, Any]]): Additional parameters for the crawl request.
idempotency_key (Optional[str]): A unique uuid key to ensure idempotency of requests.
Returns:
CrawlWatcher: An instance of CrawlWatcher to monitor the crawl job.
"""
crawl_response = self.async_crawl_url(url, params, idempotency_key)
if crawl_response['success'] and 'id' in crawl_response:
return CrawlWatcher(crawl_response['id'], self)
else:
raise Exception("Crawl job failed to start")
def map_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any: def map_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any:
""" """
Perform a map search using the Firecrawl API. Perform a map search using the Firecrawl API.
Args:
url (str): The URL to perform the map search on.
params (Optional[Dict[str, Any]]): Additional parameters for the map search.
Returns:
Any: The result of the map search, typically a dictionary containing mapping data.
""" """
if self.version == 'v0': endpoint = f'/v1/map'
raise NotImplementedError("Map is not supported in v0")
endpoint = f'/{self.version}/map'
headers = self._prepare_headers() headers = self._prepare_headers()
# Prepare the base scrape parameters with the URL # Prepare the base scrape parameters with the URL
@ -341,7 +338,7 @@ class FirecrawlApp:
return response return response
return response return response
def _monitor_job_status(self, id: str, headers: Dict[str, str], poll_interval: int, check_url: Optional[str] = None) -> Any: def _monitor_job_status(self, id: str, headers: Dict[str, str], poll_interval: int) -> Any:
""" """
Monitor the status of a crawl job until completion. Monitor the status of a crawl job until completion.
@ -349,7 +346,6 @@ class FirecrawlApp:
id (str): The ID of the crawl job. id (str): The ID of the crawl job.
headers (Dict[str, str]): The headers to include in the status check requests. headers (Dict[str, str]): The headers to include in the status check requests.
poll_interval (int): Secounds between status checks. poll_interval (int): Secounds between status checks.
check_url (Optional[str]): The URL to check for the crawl job.
Returns: Returns:
Any: The crawl results if the job is completed successfully. Any: The crawl results if the job is completed successfully.
@ -357,27 +353,14 @@ class FirecrawlApp:
Exception: If the job fails or an error occurs during status checks. Exception: If the job fails or an error occurs during status checks.
""" """
while True: while True:
api_url = '' api_url = f'{self.api_url}/v1/crawl/{id}'
if (self.version == 'v0'):
if check_url:
api_url = check_url
else:
api_url = f'{self.api_url}/v0/crawl/status/{id}'
else:
if check_url:
api_url = check_url
else:
api_url = f'{self.api_url}/v1/crawl/{id}'
status_response = self._get_request(api_url, headers) status_response = self._get_request(api_url, headers)
if status_response.status_code == 200: if status_response.status_code == 200:
status_data = status_response.json() status_data = status_response.json()
if status_data['status'] == 'completed': if status_data['status'] == 'completed':
if 'data' in status_data: if 'data' in status_data:
if self.version == 'v0': return status_data
return status_data['data']
else:
return status_data
else: else:
raise Exception('Crawl job completed but no data was returned') raise Exception('Crawl job completed but no data was returned')
elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']: elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:
@ -415,4 +398,50 @@ class FirecrawlApp:
# Raise an HTTPError with the custom message and attach the response # Raise an HTTPError with the custom message and attach the response
raise requests.exceptions.HTTPError(message, response=response) raise requests.exceptions.HTTPError(message, response=response)
class CrawlWatcher:
def __init__(self, id: str, app: FirecrawlApp):
self.id = id
self.app = app
self.data: List[Dict[str, Any]] = []
self.status = "scraping"
self.ws_url = f"{app.api_url.replace('http', 'ws')}/v1/crawl/{id}"
self.event_handlers = {
'done': [],
'error': [],
'document': []
}
async def connect(self):
async with websockets.connect(self.ws_url, extra_headers={"Authorization": f"Bearer {self.app.api_key}"}) as websocket:
await self._listen(websocket)
async def _listen(self, websocket):
async for message in websocket:
msg = json.loads(message)
await self._handle_message(msg)
def add_event_listener(self, event_type: str, handler):
if event_type in self.event_handlers:
self.event_handlers[event_type].append(handler)
def dispatch_event(self, event_type: str, detail: Dict[str, Any]):
if event_type in self.event_handlers:
for handler in self.event_handlers[event_type]:
handler(detail)
async def _handle_message(self, msg: Dict[str, Any]):
if msg['type'] == 'done':
self.status = 'completed'
self.dispatch_event('done', {'status': self.status, 'data': self.data})
elif msg['type'] == 'error':
self.status = 'failed'
self.dispatch_event('error', {'status': self.status, 'data': self.data, 'error': msg['error']})
elif msg['type'] == 'catchup':
self.status = msg['data']['status']
self.data.extend(msg['data'].get('data', []))
for doc in self.data:
self.dispatch_event('document', doc)
elif msg['type'] == 'document':
self.data.append(msg['data'])
self.dispatch_event('document', msg['data'])

4
apps/python-sdk/pyproject.toml
View File

@ -10,6 +10,10 @@ readme = {file="README.md", content-type = "text/markdown"}
requires-python = ">=3.8" requires-python = ">=3.8"
dependencies = [ dependencies = [
"requests", "requests",
"python-dotenv",
"websockets",
"asyncio",
"nest-asyncio"
] ]
authors = [{name = "Mendable.ai",email = "nick@mendable.ai"}] authors = [{name = "Mendable.ai",email = "nick@mendable.ai"}]
maintainers = [{name = "Mendable.ai",email = "nick@mendable.ai"}] maintainers = [{name = "Mendable.ai",email = "nick@mendable.ai"}]

5
apps/python-sdk/requirements.txt
View File

@ -1,3 +1,6 @@
requests requests
pytest pytest
python-dotenv python-dotenv
websockets
asyncio
nest-asyncio

3
apps/python-sdk/setup.py
View File

@ -30,6 +30,9 @@ setup(
'requests', 'requests',
'pytest', 'pytest',
'python-dotenv', 'python-dotenv',
'websockets',
'asyncio',
'nest-asyncio'
], ],
python_requires=">=3.8", python_requires=">=3.8",
classifiers=[ classifiers=[