mirror of
https://github.com/mendableai/firecrawl.git
synced 2024-11-16 03:32:22 +08:00
fix(v1): js-sdk fixed crawl type
This commit is contained in:
parent
70bff7f8fb
commit
c3158b0f98
|
@ -1,34 +1,39 @@
|
|||
import FirecrawlApp, { ScrapeResponse } from './firecrawl/src/index' //'@mendable/firecrawl-js';
|
||||
import { CrawlStatusResponse } from './firecrawl/src/index';
|
||||
import FirecrawlApp, { CrawlStatusResponse, CrawlResponse } from '@mendable/firecrawl-js';
|
||||
|
||||
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});
|
||||
|
||||
// Scrape a website:
|
||||
const scrapeResult = await app.scrapeUrl('firecrawl.dev');
|
||||
const main = async () => {
|
||||
|
||||
if (scrapeResult) {
|
||||
console.log(scrapeResult.markdown)
|
||||
}
|
||||
// Scrape a website:
|
||||
const scrapeResult = await app.scrapeUrl('firecrawl.dev');
|
||||
|
||||
// Crawl a website:
|
||||
const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludePaths: ['blog/*'], limit: 5}}, false);
|
||||
console.log(crawlResult)
|
||||
|
||||
const jobId: string = await crawlResult['jobId'];
|
||||
console.log(jobId);
|
||||
|
||||
let job: CrawlStatusResponse;
|
||||
while (true) {
|
||||
job = await app.checkCrawlStatus(jobId) as CrawlStatusResponse;
|
||||
if (job.status === 'completed') {
|
||||
break;
|
||||
if (scrapeResult) {
|
||||
console.log(scrapeResult.markdown)
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
|
||||
|
||||
// Crawl a website:
|
||||
// @ts-ignore
|
||||
const crawlResult = await app.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}, false) as CrawlResponse;
|
||||
console.log(crawlResult)
|
||||
|
||||
const id = crawlResult.id;
|
||||
console.log(id);
|
||||
|
||||
let checkStatus: CrawlStatusResponse;
|
||||
while (true) {
|
||||
checkStatus = await app.checkCrawlStatus(id);
|
||||
if (checkStatus.status === 'completed') {
|
||||
break;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
|
||||
}
|
||||
|
||||
if (checkStatus.data) {
|
||||
console.log(checkStatus.data[0].markdown);
|
||||
}
|
||||
|
||||
const mapResult = await app.mapUrl('https://firecrawl.dev');
|
||||
console.log(mapResult)
|
||||
}
|
||||
|
||||
if (job.data) {
|
||||
console.log(job.data[0].markdown);
|
||||
}
|
||||
|
||||
const mapResult = await app.mapUrl('https://firecrawl.dev');
|
||||
console.log(mapResult)
|
||||
main()
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@mendable/firecrawl-js",
|
||||
"version": "1.0.2",
|
||||
"version": "1.0.3",
|
||||
"description": "JavaScript SDK for Firecrawl API",
|
||||
"main": "build/cjs/index.js",
|
||||
"types": "types/index.d.ts",
|
||||
|
|
|
@ -165,16 +165,14 @@ export interface ScrapeResponseV0 {
|
|||
* Includes options for both scraping and mapping during a crawl.
|
||||
*/
|
||||
export interface CrawlParams {
|
||||
includePaths?: string[];
|
||||
excludePaths?: string[];
|
||||
maxDepth?: number;
|
||||
limit?: number;
|
||||
allowBackwardLinks?: boolean;
|
||||
allowExternalLinks?: boolean;
|
||||
ignoreSitemap?: boolean;
|
||||
scrapeOptions?: ScrapeParams;
|
||||
crawlerOptions?: {
|
||||
includePaths?: string[]
|
||||
excludePaths?: string[]
|
||||
maxDepth?: number
|
||||
limit?: number
|
||||
allowBackwardLinks?: boolean
|
||||
allowExternalLinks?: boolean
|
||||
ignoreSitemap?: boolean
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -11,7 +11,7 @@
|
|||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@mendable/firecrawl-js": "^0.0.36",
|
||||
"@mendable/firecrawl-js": "^1.0.3",
|
||||
"axios": "^1.6.8",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.4.5",
|
||||
|
|
Loading…
Reference in New Issue
Block a user