Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(crawler): make sure includes/excludes is an array

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-22 13:18:26 +02:00
parent 508568f943
commit fbbc3878f1
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -53,8 +53,8 @@ export class WebCrawler {
this.jobId = jobId; this.jobId = jobId;
this.initialUrl = initialUrl; this.initialUrl = initialUrl;
this.baseUrl = new URL(initialUrl).origin; this.baseUrl = new URL(initialUrl).origin;
this.includes = includes ?? []; this.includes = Array.isArray(includes) ? includes : [];
this.excludes = excludes ?? []; this.excludes = Array.isArray(excludes) ? excludes : [];
this.limit = limit; this.limit = limit;
this.robotsTxtUrl = `${this.baseUrl}/robots.txt`; this.robotsTxtUrl = `${this.baseUrl}/robots.txt`;
this.robots = robotsParser(this.robotsTxtUrl, ""); this.robots = robotsParser(this.robotsTxtUrl, "");