Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(WebScraper): fatal error handler triggering for 404s

Browse Source
This commit is contained in:
Gergő Móricz 2024-09-23 18:33:10 +02:00
parent 4721aa1687
commit d2f7031069
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -421,7 +421,9 @@ export async function scrapSingleUrl(
// }
}
if (!text) {
// NOTE: This exception for status codes may only work with fire-engine. In lieu of better error management,
// it's the best we can do. - mogery
if (!text && !Object.values(errors).some(x => x.startsWith("Request failed with status code "))) {
throw new Error(`All scraping methods failed for URL: ${urlToScrap}`);
}