diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 93fb0ce7..d0b7269f 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -421,7 +421,9 @@ export async function scrapSingleUrl( // } } - if (!text) { + // NOTE: This exception for status codes may only work with fire-engine. In lieu of better error management, + // it's the best we can do. - mogery + if (!text && !Object.values(errors).some(x => x.startsWith("Request failed with status code "))) { throw new Error(`All scraping methods failed for URL: ${urlToScrap}`); }