Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(scrapeURL/pdf): retry

Browse Source
This commit is contained in:
Gergő Móricz 2024-11-12 22:26:36 +01:00
parent 9ace2ad071
commit 7081beff1f
2 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
View File

@ -62,6 +62,8 @@ async function scrapePDFWithLlamaParse(meta: Meta, tempFilePath: string): Promis
schema: z.object({
markdown: z.string(),
}),
tryCount: 16,
tryCooldown: 250,
});
return {

7
apps/api/src/scraper/scrapeURL/lib/fetch.ts
View File

@ -15,6 +15,7 @@ export type RobustFetchParams<Schema extends z.Schema<any>> = {
ignoreFailure?: boolean;
requestId?: string;
tryCount?: number;
tryCooldown?: number;
};
export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer<Schema>>({
@ -28,8 +29,9 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer
ignoreFailure = false,
requestId = uuid(),
tryCount = 1,
tryCooldown,
}: RobustFetchParams<Schema>): Promise<Output> {
const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure, tryCount };
const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure, tryCount, tryCooldown };
let request: Response;
try {
@ -86,6 +88,9 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer
if (request.status >= 300) {
if (tryCount > 1) {
logger.debug("Request sent failure status, trying " + (tryCount - 1) + " more times", { params, request, response, requestId });
if (tryCooldown !== undefined) {
await new Promise((resolve) => setTimeout(() => resolve(null), tryCooldown));
}
return await robustFetch({
...params,
requestId,