From c3158b0f9864637bde67a44728ae4f91c5368e22 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:27:29 -0300 Subject: [PATCH] fix(v1): js-sdk fixed crawl type --- apps/js-sdk/example.ts | 57 ++++++++++++++++-------------- apps/js-sdk/firecrawl/package.json | 2 +- apps/js-sdk/firecrawl/src/index.ts | 16 ++++----- apps/js-sdk/package.json | 2 +- 4 files changed, 40 insertions(+), 37 deletions(-) diff --git a/apps/js-sdk/example.ts b/apps/js-sdk/example.ts index 9a4d840c..f8d7d5d9 100644 --- a/apps/js-sdk/example.ts +++ b/apps/js-sdk/example.ts @@ -1,34 +1,39 @@ -import FirecrawlApp, { ScrapeResponse } from './firecrawl/src/index' //'@mendable/firecrawl-js'; -import { CrawlStatusResponse } from './firecrawl/src/index'; +import FirecrawlApp, { CrawlStatusResponse, CrawlResponse } from '@mendable/firecrawl-js'; const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); -// Scrape a website: -const scrapeResult = await app.scrapeUrl('firecrawl.dev'); +const main = async () => { -if (scrapeResult) { - console.log(scrapeResult.markdown) -} + // Scrape a website: + const scrapeResult = await app.scrapeUrl('firecrawl.dev'); -// Crawl a website: -const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludePaths: ['blog/*'], limit: 5}}, false); -console.log(crawlResult) - -const jobId: string = await crawlResult['jobId']; -console.log(jobId); - -let job: CrawlStatusResponse; -while (true) { - job = await app.checkCrawlStatus(jobId) as CrawlStatusResponse; - if (job.status === 'completed') { - break; + if (scrapeResult) { + console.log(scrapeResult.markdown) } - await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second + + // Crawl a website: + // @ts-ignore + const crawlResult = await app.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}, false) as CrawlResponse; + console.log(crawlResult) + + const id = crawlResult.id; + console.log(id); + + let checkStatus: CrawlStatusResponse; + while (true) { + checkStatus = await app.checkCrawlStatus(id); + if (checkStatus.status === 'completed') { + break; + } + await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second + } + + if (checkStatus.data) { + console.log(checkStatus.data[0].markdown); + } + + const mapResult = await app.mapUrl('https://firecrawl.dev'); + console.log(mapResult) } -if (job.data) { - console.log(job.data[0].markdown); -} - -const mapResult = await app.mapUrl('https://firecrawl.dev'); -console.log(mapResult) +main() \ No newline at end of file diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 9f3fd6a9..6eb37a22 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.0.2", + "version": "1.0.3", "description": "JavaScript SDK for Firecrawl API", "main": "build/cjs/index.js", "types": "types/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 4f3f820f..cb2a0e4f 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -165,16 +165,14 @@ export interface ScrapeResponseV0 { * Includes options for both scraping and mapping during a crawl. */ export interface CrawlParams { + includePaths?: string[]; + excludePaths?: string[]; + maxDepth?: number; + limit?: number; + allowBackwardLinks?: boolean; + allowExternalLinks?: boolean; + ignoreSitemap?: boolean; scrapeOptions?: ScrapeParams; - crawlerOptions?: { - includePaths?: string[] - excludePaths?: string[] - maxDepth?: number - limit?: number - allowBackwardLinks?: boolean - allowExternalLinks?: boolean - ignoreSitemap?: boolean - }; } /** diff --git a/apps/js-sdk/package.json b/apps/js-sdk/package.json index 2be17886..b5d919f4 100644 --- a/apps/js-sdk/package.json +++ b/apps/js-sdk/package.json @@ -11,7 +11,7 @@ "author": "", "license": "ISC", "dependencies": { - "@mendable/firecrawl-js": "^0.0.36", + "@mendable/firecrawl-js": "^1.0.3", "axios": "^1.6.8", "ts-node": "^10.9.2", "typescript": "^5.4.5",