Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-15 19:22:19 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

added .inc files and forced lower case comparison

Browse Source
This commit is contained in:
rafaelsideguide 2024-07-31 09:28:43 -03:00
parent 4ab7fb007a
commit f48ff36b32
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -164,7 +164,7 @@ export class WebCrawler {
concurrencyLimit,
inProgress
);
if (
urls.length === 0 &&
this.filterLinks([this.initialUrl], limit, this.maxCrawledDepth).length > 0
@ -420,9 +420,10 @@ export class WebCrawler {
".woff",
".ttf",
".woff2",
".webp"
".webp",
".inc"
];
return fileExtensions.some((ext) => url.endsWith(ext));
return fileExtensions.some((ext) => url.toLowerCase().endsWith(ext));
}
private isSocialMediaOrEmail(url: string): boolean {