Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Update single_url.ts

Browse Source
This commit is contained in:
Nicolas 2024-07-25 17:51:41 -04:00
parent 56042d090c
commit 01fab6e036
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -26,10 +26,10 @@ export const baseScrapers = [
"fire-engine", "fire-engine",
"fire-engine;chrome-cdp", "fire-engine;chrome-cdp",
"scrapingBee", "scrapingBee",
"playwright", process.env.USE_DB_AUTHENTICATION ? undefined : "playwright",
"scrapingBeeLoad", "scrapingBeeLoad",
"fetch", "fetch",
] as const; ].filter(Boolean);
export async function generateRequestParams( export async function generateRequestParams(
url: string, url: string,
@ -95,11 +95,11 @@ function getScrapingFallbackOrder(
if (isWaitPresent || isScreenshotPresent || isHeadersPresent) { if (isWaitPresent || isScreenshotPresent || isHeadersPresent) {
defaultOrder = [ defaultOrder = [
"fire-engine", "fire-engine",
"playwright", process.env.USE_DB_AUTHENTICATION ? undefined : "playwright",
...defaultOrder.filter( ...defaultOrder.filter(
(scraper) => scraper !== "fire-engine" && scraper !== "playwright" (scraper) => scraper !== "fire-engine" && scraper !== "playwright"
), ),
]; ].filter(Boolean);
} }
const filteredDefaultOrder = defaultOrder.filter( const filteredDefaultOrder = defaultOrder.filter(