Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #296 from NeevJewalkar/sdk-update

Browse Source 
Added local host support for the javascript SDK
This commit is contained in:
Nicolas 2024-06-18 13:03:37 -04:00 committed by GitHub
commit a3115004af
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 55 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/js-sdk/firecrawl/README.md
View File

@ -176,6 +176,11 @@ async function checkStatusExample(jobId) {
checkStatusExample('your_job_id_here'); checkStatusExample('your_job_id_here');
``` ```
## Running Locally
To use the SDK when running Firecrawl locally, you can change the initial Firecrawl app instance to:
```js
const app = new FirecrawlApp({ apiKey: "YOUR_API_KEY", apiUrl: "http://localhost:3002" });
```
## Error Handling ## Error Handling

4
apps/js-sdk/firecrawl/build/index.js
View File

@ -18,9 +18,9 @@ export default class FirecrawlApp {
* Initializes a new instance of the FirecrawlApp class. * Initializes a new instance of the FirecrawlApp class.
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
*/ */
constructor({ apiKey = null }) { constructor({ apiKey = null, apiUrl = null }) {
this.apiUrl = "https://api.firecrawl.dev";
this.apiKey = apiKey || ""; this.apiKey = apiKey || "";
this.apiUrl = apiUrl || "https://api.firecrawl.dev";
if (!this.apiKey) { if (!this.apiKey) {
throw new Error("No API key provided"); throw new Error("No API key provided");
} }

4
apps/js-sdk/firecrawl/package-lock.json generated
View File

@ -1,12 +1,12 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "0.0.22", "version": "0.0.26",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "0.0.22", "version": "0.0.26",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"axios": "^1.6.8", "axios": "^1.6.8",

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "0.0.27", "version": "0.0.28",
"description": "JavaScript SDK for Firecrawl API", "description": "JavaScript SDK for Firecrawl API",
"main": "build/index.js", "main": "build/index.js",
"types": "types/index.d.ts", "types": "types/index.d.ts",

83
apps/js-sdk/firecrawl/src/index.ts
View File

@ -73,62 +73,63 @@ export interface FirecrawlDocument {
* Response interface for scraping operations. * Response interface for scraping operations.
*/ */
export interface ScrapeResponse { export interface ScrapeResponse {
success: boolean; success: boolean;
data?: FirecrawlDocument; data?: FirecrawlDocument;
error?: string; error?: string;
} }
/** /**
* Response interface for searching operations. * Response interface for searching operations.
*/ */
export interface SearchResponse { export interface SearchResponse {
success: boolean; success: boolean;
data?: FirecrawlDocument[]; data?: FirecrawlDocument[];
error?: string; error?: string;
} }
/** /**
* Response interface for crawling operations. * Response interface for crawling operations.
*/ */
export interface CrawlResponse { export interface CrawlResponse {
success: boolean; success: boolean;
jobId?: string; jobId?: string;
data?: FirecrawlDocument[]; data?: FirecrawlDocument[];
error?: string; error?: string;
} }
/** /**
* Response interface for job status checks. * Response interface for job status checks.
*/ */
export interface JobStatusResponse { export interface JobStatusResponse {
success: boolean; success: boolean;
status: string; status: string;
jobId?: string; jobId?: string;
data?: FirecrawlDocument[]; data?: FirecrawlDocument[];
partial_data?: FirecrawlDocument[]; partial_data?: FirecrawlDocument[];
error?: string; error?: string;
} }
/** /**
* Generic parameter interface. * Generic parameter interface.
*/ */
export interface Params { export interface Params {
[key: string]: any; [key: string]: any;
extractorOptions?: { extractorOptions?: {
extractionSchema: z.ZodSchema | any; extractionSchema: z.ZodSchema | any;
mode?: "llm-extraction"; mode?: "llm-extraction";
extractionPrompt?: string; extractionPrompt?: string;
}; };
} }
/** /**
* Main class for interacting with the Firecrawl API. * Main class for interacting with the Firecrawl API.
*/ */
export default class FirecrawlApp { export default class FirecrawlApp {
private apiKey: string; private apiKey: string;
private apiUrl: string = "https://api.firecrawl.dev"; private apiUrl: string;
/** /**
* Initializes a new instance of the FirecrawlApp class. * Initializes a new instance of the FirecrawlApp class.
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
*/ */
constructor({ apiKey = null }: FirecrawlAppConfig) { constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
this.apiKey = apiKey || ""; this.apiKey = apiKey || "";
this.apiUrl = apiUrl || "https://api.firecrawl.dev";
if (!this.apiKey) { if (!this.apiKey) {
throw new Error("No API key provided"); throw new Error("No API key provided");
} }
@ -168,7 +169,7 @@ export default class FirecrawlApp {
const response: AxiosResponse = await axios.post( const response: AxiosResponse = await axios.post(
this.apiUrl + "/v0/scrape", this.apiUrl + "/v0/scrape",
jsonData, jsonData,
{ headers }, { headers }
); );
if (response.status === 200) { if (response.status === 200) {
const responseData = response.data; const responseData = response.data;
@ -287,7 +288,9 @@ export default class FirecrawlApp {
success: true, success: true,
status: response.data.status, status: response.data.status,
data: response.data.data, data: response.data.data,
partial_data: !response.data.data ? response.data.partial_data : undefined, partial_data: !response.data.data
? response.data.partial_data
: undefined,
}; };
} else { } else {
this.handleError(response, "check crawl status"); this.handleError(response, "check crawl status");
@ -308,10 +311,10 @@ export default class FirecrawlApp {
*/ */
prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders { prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
return { return {
'Content-Type': 'application/json', "Content-Type": "application/json",
'Authorization': `Bearer ${this.apiKey}`, Authorization: `Bearer ${this.apiKey}`,
...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}), ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
} as AxiosRequestHeaders & { 'x-idempotency-key'?: string }; } as AxiosRequestHeaders & { "x-idempotency-key"?: string };
} }
/** /**
@ -373,7 +376,9 @@ export default class FirecrawlApp {
if (checkInterval < 2) { if (checkInterval < 2) {
checkInterval = 2; checkInterval = 2;
} }
await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again await new Promise((resolve) =>
setTimeout(resolve, checkInterval * 1000)
); // Wait for the specified timeout before checking again
} else { } else {
throw new Error( throw new Error(
`Crawl job failed or was stopped. Status: ${statusData.status}` `Crawl job failed or was stopped. Status: ${statusData.status}`

2
apps/js-sdk/firecrawl/types/index.d.ts vendored
View File

@ -121,7 +121,7 @@ export default class FirecrawlApp {
* Initializes a new instance of the FirecrawlApp class. * Initializes a new instance of the FirecrawlApp class.
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
*/ */
constructor({ apiKey }: FirecrawlAppConfig); constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
/** /**
* Scrapes a URL using the Firecrawl API. * Scrapes a URL using the Firecrawl API.
* @param {string} url - The URL to scrape. * @param {string} url - The URL to scrape.

40
apps/js-sdk/package-lock.json generated
View File

@ -11,10 +11,8 @@
"dependencies": { "dependencies": {
"@mendable/firecrawl-js": "^0.0.19", "@mendable/firecrawl-js": "^0.0.19",
"axios": "^1.6.8", "axios": "^1.6.8",
"dotenv": "^16.4.5",
"ts-node": "^10.9.2", "ts-node": "^10.9.2",
"typescript": "^5.4.5", "typescript": "^5.4.5",
"uuid": "^9.0.1",
"zod": "^3.23.8" "zod": "^3.23.8"
}, },
"devDependencies": { "devDependencies": {
@ -452,15 +450,6 @@
"resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
"integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==" "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA=="
}, },
"node_modules/@types/node": {
"version": "20.12.11",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.11.tgz",
"integrity": "sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw==",
"peer": true,
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/acorn": { "node_modules/acorn": {
"version": "8.11.3", "version": "8.11.3",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
@ -532,17 +521,6 @@
"node": ">=0.3.1" "node": ">=0.3.1"
} }
}, },
"node_modules/dotenv": {
"version": "16.4.5",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz",
"integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/esbuild": { "node_modules/esbuild": {
"version": "0.20.2", "version": "0.20.2",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz",
@ -750,24 +728,6 @@
"node": ">=14.17" "node": ">=14.17"
} }
}, },
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"peer": true
},
"node_modules/uuid": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/v8-compile-cache-lib": { "node_modules/v8-compile-cache-lib": {
"version": "3.0.1", "version": "3.0.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",