Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #296 from NeevJewalkar/sdk-update

Browse Source 
Added local host support for the javascript SDK
This commit is contained in:
Nicolas 2024-06-18 13:03:37 -04:00 committed by GitHub
commit a3115004af
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 55 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/js-sdk/firecrawl/README.md
View File

@ -176,6 +176,11 @@ async function checkStatusExample(jobId) {
checkStatusExample('your_job_id_here');
```
## Running Locally
To use the SDK when running Firecrawl locally, you can change the initial Firecrawl app instance to:
```js
const app = new FirecrawlApp({ apiKey: "YOUR_API_KEY", apiUrl: "http://localhost:3002" });
```
## Error Handling

4
apps/js-sdk/firecrawl/build/index.js
View File

@ -18,9 +18,9 @@ export default class FirecrawlApp {
* Initializes a new instance of the FirecrawlApp class.
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
*/
constructor({ apiKey = null }) {
this.apiUrl = "https://api.firecrawl.dev";
constructor({ apiKey = null, apiUrl = null }) {
this.apiKey = apiKey || "";
this.apiUrl = apiUrl || "https://api.firecrawl.dev";
if (!this.apiKey) {
throw new Error("No API key provided");
}

4
apps/js-sdk/firecrawl/package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "@mendable/firecrawl-js",
"version": "0.0.22",
"version": "0.0.26",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@mendable/firecrawl-js",
"version": "0.0.22",
"version": "0.0.26",
"license": "MIT",
"dependencies": {
"axios": "^1.6.8",

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
"version": "0.0.27",
"version": "0.0.28",
"description": "JavaScript SDK for Firecrawl API",
"main": "build/index.js",
"types": "types/index.d.ts",

83
apps/js-sdk/firecrawl/src/index.ts
View File

@ -73,62 +73,63 @@ export interface FirecrawlDocument {
* Response interface for scraping operations.
*/
export interface ScrapeResponse {
success: boolean;
data?: FirecrawlDocument;
error?: string;
success: boolean;
data?: FirecrawlDocument;
error?: string;
}
/**
* Response interface for searching operations.
*/
* Response interface for searching operations.
*/
export interface SearchResponse {
success: boolean;
data?: FirecrawlDocument[];
error?: string;
success: boolean;
data?: FirecrawlDocument[];
error?: string;
}
/**
* Response interface for crawling operations.
*/
* Response interface for crawling operations.
*/
export interface CrawlResponse {
success: boolean;
jobId?: string;
data?: FirecrawlDocument[];
error?: string;
success: boolean;
jobId?: string;
data?: FirecrawlDocument[];
error?: string;
}
/**
* Response interface for job status checks.
*/
* Response interface for job status checks.
*/
export interface JobStatusResponse {
success: boolean;
status: string;
jobId?: string;
data?: FirecrawlDocument[];
partial_data?: FirecrawlDocument[];
error?: string;
success: boolean;
status: string;
jobId?: string;
data?: FirecrawlDocument[];
partial_data?: FirecrawlDocument[];
error?: string;
}
/**
* Generic parameter interface.
*/
* Generic parameter interface.
*/
export interface Params {
[key: string]: any;
extractorOptions?: {
extractionSchema: z.ZodSchema | any;
mode?: "llm-extraction";
extractionPrompt?: string;
};
[key: string]: any;
extractorOptions?: {
extractionSchema: z.ZodSchema | any;
mode?: "llm-extraction";
extractionPrompt?: string;
};
}
/**
* Main class for interacting with the Firecrawl API.
*/
export default class FirecrawlApp {
private apiKey: string;
private apiUrl: string = "https://api.firecrawl.dev";
private apiUrl: string;
/**
* Initializes a new instance of the FirecrawlApp class.
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
*/
constructor({ apiKey = null }: FirecrawlAppConfig) {
constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
this.apiKey = apiKey || "";
this.apiUrl = apiUrl || "https://api.firecrawl.dev";
if (!this.apiKey) {
throw new Error("No API key provided");
}
@ -168,7 +169,7 @@ export default class FirecrawlApp {
const response: AxiosResponse = await axios.post(
this.apiUrl + "/v0/scrape",
jsonData,
{ headers },
{ headers }
);
if (response.status === 200) {
const responseData = response.data;
@ -287,7 +288,9 @@ export default class FirecrawlApp {
success: true,
status: response.data.status,
data: response.data.data,
partial_data: !response.data.data ? response.data.partial_data : undefined,
partial_data: !response.data.data
? response.data.partial_data
: undefined,
};
} else {
this.handleError(response, "check crawl status");
@ -308,10 +311,10 @@ export default class FirecrawlApp {
*/
prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
return {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`,
...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}),
} as AxiosRequestHeaders & { 'x-idempotency-key'?: string };
"Content-Type": "application/json",
Authorization: `Bearer ${this.apiKey}`,
...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
} as AxiosRequestHeaders & { "x-idempotency-key"?: string };
}
/**
@ -373,7 +376,9 @@ export default class FirecrawlApp {
if (checkInterval < 2) {
checkInterval = 2;
}
await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
await new Promise((resolve) =>
setTimeout(resolve, checkInterval * 1000)
); // Wait for the specified timeout before checking again
} else {
throw new Error(
`Crawl job failed or was stopped. Status: ${statusData.status}`

2
apps/js-sdk/firecrawl/types/index.d.ts vendored
View File

@ -121,7 +121,7 @@ export default class FirecrawlApp {
* Initializes a new instance of the FirecrawlApp class.
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
*/
constructor({ apiKey }: FirecrawlAppConfig);
constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
/**
* Scrapes a URL using the Firecrawl API.
* @param {string} url - The URL to scrape.

40
apps/js-sdk/package-lock.json generated
View File

@ -11,10 +11,8 @@
"dependencies": {
"@mendable/firecrawl-js": "^0.0.19",
"axios": "^1.6.8",
"dotenv": "^16.4.5",
"ts-node": "^10.9.2",
"typescript": "^5.4.5",
"uuid": "^9.0.1",
"zod": "^3.23.8"
},
"devDependencies": {
@ -452,15 +450,6 @@
"resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
"integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA=="
},
"node_modules/@types/node": {
"version": "20.12.11",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.11.tgz",
"integrity": "sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw==",
"peer": true,
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/acorn": {
"version": "8.11.3",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
@ -532,17 +521,6 @@
"node": ">=0.3.1"
}
},
"node_modules/dotenv": {
"version": "16.4.5",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz",
"integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/esbuild": {
"version": "0.20.2",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz",
@ -750,24 +728,6 @@
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"peer": true
},
"node_modules/uuid": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/v8-compile-cache-lib": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",