Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-15 19:22:19 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat(batchScrape): handle timeout

Browse Source
This commit is contained in:
Móricz Gergő 2024-11-12 12:42:39 +01:00
parent f6db9f1428
commit 91f52287db
1 changed files with 24 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/api/src/services/queue-worker.ts
View File

@ -276,10 +276,18 @@ async function processJob(job: Job & { id: string }, token: string) {
});
const start = Date.now();
const pipeline = await startWebScraperPipeline({
const pipeline = await Promise.race([
startWebScraperPipeline({
job,
token,
});
}),
...(job.data.scrapeOptions.timeout !== undefined ? [
(async () => {
await sleep(job.data.scrapeOptions.timeout);
throw new Error("timeout")
})(),
] : [])
]);
if (!pipeline.success) {
// TODO: let's Not do this
@ -486,6 +494,9 @@ async function processJob(job: Job & { id: string }, token: string) {
logger.info(`🐂 Job done ${job.id}`);
return data;
} catch (error) {
const isEarlyTimeout = error instanceof Error && error.message === "timeout";
if (!isEarlyTimeout) {
logger.error(`🐂 Job errored ${job.id} - ${error}`);
Sentry.captureException(error, {
@ -493,6 +504,9 @@ async function processJob(job: Job & { id: string }, token: string) {
job: job.id,
},
});
} else {
logger.error(`🐂 Job timed out ${job.id}`);
}
if (error instanceof CustomError) {
// Here we handle the error, then save the failed job