Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #494 from mendableai/nsc/website-param-fixes
Some checks failed
Fly Deploy / Pre-deploy checks (push) Has been cancelled
Details
Fly Deploy / Test Suite (push) Has been cancelled
Details
Fly Deploy / Python SDK Tests (push) Has been cancelled
Details
Fly Deploy / JavaScript SDK Tests (push) Has been cancelled
Details
Fly Deploy / Deploy app (push) Has been cancelled
Details
Fly Deploy / Build and publish Python SDK (push) Has been cancelled
Details
Fly Deploy / Build and publish JavaScript SDK (push) Has been cancelled
Details

Browse Source 
Custom engine params fix
This commit is contained in:
Nicolas 2024-08-02 19:29:03 -04:00 committed by GitHub
commit 789c6cf5d7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
View File

@ -47,7 +47,7 @@ export async function scrapWithFireEngine({
try {
const reqParams = await generateRequestParams(url);
const waitParam = reqParams["params"]?.wait ?? waitFor;
const engineParam = reqParams["params"]?.engine ?? fireEngineOptions?.engine ?? "playwright";
const engineParam = reqParams["params"]?.engine ?? reqParams["params"]?.fireEngineOptions?.engine ?? fireEngineOptions?.engine ?? "playwright";
const screenshotParam = reqParams["params"]?.screenshot ?? screenshot;
const fireEngineOptionsParam : FireEngineOptions = reqParams["params"]?.fireEngineOptions ?? fireEngineOptions;
@ -64,6 +64,7 @@ export async function scrapWithFireEngine({
`⛏️ Fire-Engine (${engine}): Scraping ${url} | params: { wait: ${waitParam}, screenshot: ${screenshotParam}, method: ${fireEngineOptionsParam?.method ?? "null"} }`
);
const response = await axios.post(
process.env.FIRE_ENGINE_BETA_URL + endpoint,
{

2
apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
View File

@ -236,8 +236,6 @@ export const urlSpecificParams = {
defaultScraper: "fire-engine",
params:{
fireEngineOptions:{
mobileProxy: true,
method: "get",
engine: "chrome-cdp",
},
},