Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Ensuring USE_DB_AUTHENTICATION is true in single URL scraper.

Browse Source
This commit is contained in:
Kevin Swiber 2024-08-09 15:29:18 -07:00
parent 5a778f2c22
commit ba2af74adf
No known key found for this signature in database
GPG Key ID: D31500F61C91A451
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -23,11 +23,13 @@ import { clientSideError } from "../../strings";
dotenv.config(); dotenv.config();
const useDatabaseAuth = process.env.USE_DB_AUTHENTICATION === "true";
export const baseScrapers = [ export const baseScrapers = [
"fire-engine", "fire-engine",
"fire-engine;chrome-cdp", "fire-engine;chrome-cdp",
"scrapingBee", "scrapingBee",
process.env.USE_DB_AUTHENTICATION ? undefined : "playwright", useDatabaseAuth ? undefined : "playwright",
"scrapingBeeLoad", "scrapingBeeLoad",
"fetch", "fetch",
].filter(Boolean); ].filter(Boolean);
@ -85,10 +87,10 @@ function getScrapingFallbackOrder(
}); });
let defaultOrder = [ let defaultOrder = [
!process.env.USE_DB_AUTHENTICATION ? undefined : "fire-engine", !useDatabaseAuth ? undefined : "fire-engine",
!process.env.USE_DB_AUTHENTICATION ? undefined : "fire-engine;chrome-cdp", !useDatabaseAuth ? undefined : "fire-engine;chrome-cdp",
"scrapingBee", "scrapingBee",
process.env.USE_DB_AUTHENTICATION ? undefined : "playwright", useDatabaseAuth ? undefined : "playwright",
"scrapingBeeLoad", "scrapingBeeLoad",
"fetch", "fetch",
].filter(Boolean); ].filter(Boolean);
@ -96,7 +98,7 @@ function getScrapingFallbackOrder(
if (isWaitPresent || isScreenshotPresent || isHeadersPresent) { if (isWaitPresent || isScreenshotPresent || isHeadersPresent) {
defaultOrder = [ defaultOrder = [
"fire-engine", "fire-engine",
process.env.USE_DB_AUTHENTICATION ? undefined : "playwright", useDatabaseAuth ? undefined : "playwright",
...defaultOrder.filter( ...defaultOrder.filter(
(scraper) => scraper !== "fire-engine" && scraper !== "playwright" (scraper) => scraper !== "fire-engine" && scraper !== "playwright"
), ),