fix(v1): js-sdk fixed crawl type

This commit is contained in:
rafaelsideguide 2024-08-28 16:27:29 -03:00
parent 70bff7f8fb
commit c3158b0f98
4 changed files with 40 additions and 37 deletions

View File

@ -1,34 +1,39 @@
import FirecrawlApp, { ScrapeResponse } from './firecrawl/src/index' //'@mendable/firecrawl-js';
import { CrawlStatusResponse } from './firecrawl/src/index';
import FirecrawlApp, { CrawlStatusResponse, CrawlResponse } from '@mendable/firecrawl-js';
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});
// Scrape a website:
const scrapeResult = await app.scrapeUrl('firecrawl.dev');
const main = async () => {
if (scrapeResult) {
// Scrape a website:
const scrapeResult = await app.scrapeUrl('firecrawl.dev');
if (scrapeResult) {
console.log(scrapeResult.markdown)
}
}
// Crawl a website:
const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludePaths: ['blog/*'], limit: 5}}, false);
console.log(crawlResult)
// Crawl a website:
// @ts-ignore
const crawlResult = await app.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}, false) as CrawlResponse;
console.log(crawlResult)
const jobId: string = await crawlResult['jobId'];
console.log(jobId);
const id = crawlResult.id;
console.log(id);
let job: CrawlStatusResponse;
while (true) {
job = await app.checkCrawlStatus(jobId) as CrawlStatusResponse;
if (job.status === 'completed') {
let checkStatus: CrawlStatusResponse;
while (true) {
checkStatus = await app.checkCrawlStatus(id);
if (checkStatus.status === 'completed') {
break;
}
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
}
if (checkStatus.data) {
console.log(checkStatus.data[0].markdown);
}
const mapResult = await app.mapUrl('https://firecrawl.dev');
console.log(mapResult)
}
if (job.data) {
console.log(job.data[0].markdown);
}
const mapResult = await app.mapUrl('https://firecrawl.dev');
console.log(mapResult)
main()

View File

@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
"version": "1.0.2",
"version": "1.0.3",
"description": "JavaScript SDK for Firecrawl API",
"main": "build/cjs/index.js",
"types": "types/index.d.ts",

View File

@ -165,16 +165,14 @@ export interface ScrapeResponseV0 {
* Includes options for both scraping and mapping during a crawl.
*/
export interface CrawlParams {
includePaths?: string[];
excludePaths?: string[];
maxDepth?: number;
limit?: number;
allowBackwardLinks?: boolean;
allowExternalLinks?: boolean;
ignoreSitemap?: boolean;
scrapeOptions?: ScrapeParams;
crawlerOptions?: {
includePaths?: string[]
excludePaths?: string[]
maxDepth?: number
limit?: number
allowBackwardLinks?: boolean
allowExternalLinks?: boolean
ignoreSitemap?: boolean
};
}
/**

View File

@ -11,7 +11,7 @@
"author": "",
"license": "ISC",
"dependencies": {
"@mendable/firecrawl-js": "^0.0.36",
"@mendable/firecrawl-js": "^1.0.3",
"axios": "^1.6.8",
"ts-node": "^10.9.2",
"typescript": "^5.4.5",