Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-15 19:22:19 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat(scrapeURL): Add retry logic to robustFetch
Some checks are pending
STAGING Deploy Images to GHCR / push-app-image (push) Waiting to run
Details

Browse Source
This commit is contained in:
Móricz Gergő 2024-11-05 13:51:14 +01:00
parent cd534326ba
commit 8b69ccb7ff
2 changed files with 35 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
View File

@ -85,6 +85,7 @@ export async function fireEngineScrape<Engine extends FireEngineScrapeRequestChr
body: request,
logger: logger.child({ method: "fireEngineScrape/robustFetch" }),
schema,
tryCount: 3,
}
);
});

47
apps/api/src/scraper/scrapeURL/lib/fetch.ts
View File

@ -15,6 +15,7 @@ export type RobustFetchParams<Schema extends z.Schema<any>> = {
ignoreResponse?: boolean;
ignoreFailure?: boolean;
requestId?: string;
tryCount?: number;
};
export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer<Schema>>({
@ -27,8 +28,9 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer
ignoreResponse = false,
ignoreFailure = false,
requestId = uuid(),
tryCount = 1,
}: RobustFetchParams<Schema>): Promise<Output> {
const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure };
const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure, tryCount };
logger.debug("Sending request...", { params, requestId });
@ -53,12 +55,22 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer
} catch (error) {
if (!ignoreFailure) {
Sentry.captureException(error);
logger.debug("Request failed", { params, error, requestId });
throw new Error("Request failed", {
cause: {
params, requestId, error,
},
});
if (tryCount > 1) {
logger.debug("Request failed, trying " + (tryCount - 1) + " more times", { params, error, requestId });
return await robustFetch({
...params,
requestId,
tryCount: tryCount - 1,
});
} else {
logger.debug("Request failed", { params, error, requestId });
throw new Error("Request failed", {
cause: {
params, requestId, error,
},
});
}
} else {
return null as Output;
}
@ -75,12 +87,21 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer
};
if (request.status >= 300) {
logger.debug("Request sent failure status", { params, request, response, requestId });
throw new Error("Request sent failure status", {
cause: {
params, request, response, requestId,
},
});
if (tryCount > 1) {
logger.debug("Request sent failure status, trying " + (tryCount - 1) + " more times", { params, request, response, requestId });
return await robustFetch({
...params,
requestId,
tryCount: tryCount - 1,
});
} else {
logger.debug("Request sent failure status", { params, request, response, requestId });
throw new Error("Request sent failure status", {
cause: {
params, request, response, requestId,
},
});
}
}
let data: Output;