Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

bugfix includes excludes

Browse Source
This commit is contained in:
rafaelsideguide 2024-08-09 14:30:41 -03:00
parent bf1d325aa9
commit 0591000b64
1 changed files with 20 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
apps/api/src/scraper/WebScraper/index.ts
View File

@ -168,11 +168,29 @@ export class WebScraperDataProvider {
private async handleCrawlMode( private async handleCrawlMode(
inProgress?: (progress: Progress) => void inProgress?: (progress: Progress) => void
): Promise<Document[]> { ): Promise<Document[]> {
let includes: string[];
if (Array.isArray(this.includes)) {
if (this.includes[0] != "") {
includes = this.includes;
}
} else {
includes = this.includes.split(',');
}
let excludes: string[];
if (Array.isArray(this.excludes)) {
if (this.excludes[0] != "") {
excludes = this.excludes;
}
} else {
excludes = this.excludes.split(',');
}
const crawler = new WebCrawler({ const crawler = new WebCrawler({
jobId: this.jobId, jobId: this.jobId,
initialUrl: this.urls[0], initialUrl: this.urls[0],
includes: Array.isArray(this.includes) ? this.includes : this.includes.split(','), includes,
excludes: Array.isArray(this.excludes) ? this.excludes : this.excludes.split(','), excludes,
maxCrawledLinks: this.maxCrawledLinks, maxCrawledLinks: this.maxCrawledLinks,
maxCrawledDepth: getAdjustedMaxDepth(this.urls[0], this.maxCrawledDepth), maxCrawledDepth: getAdjustedMaxDepth(this.urls[0], this.maxCrawledDepth),
limit: this.limit, limit: this.limit,