Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(queue-worker): variable name collision

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-14 22:02:05 +02:00
parent b8ec40dd72
commit 8a5cad72f6
1 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
apps/api/src/services/queue-worker.ts
View File

@ -175,7 +175,7 @@ async function processJob(job: Job, token: string) {
const crawler = crawlToCrawler(job.data.crawl_id, sc); const crawler = crawlToCrawler(job.data.crawl_id, sc);
const links = crawler.filterLinks((data.docs[0].linksOnPage as string[]) const links = crawler.filterLinks((data.docs[0].linksOnPage as string[])
.map(href => crawler.filterURL(href, sc.originUrl)) .map(href => crawler.filterURL(href.trim(), sc.originUrl))
.filter(x => x !== null), .filter(x => x !== null),
Infinity, Infinity,
sc.crawlerOptions?.maxDepth ?? 10 sc.crawlerOptions?.maxDepth ?? 10
@ -228,12 +228,12 @@ async function processJob(job: Job, token: string) {
const jobStatuses = await Promise.all(jobs.map(x => x.getState())); const jobStatuses = await Promise.all(jobs.map(x => x.getState()));
const jobStatus = sc.cancelled ? "failed" : jobStatuses.every(x => x === "completed") ? "completed" : jobStatuses.some(x => x === "failed") ? "failed" : "active"; const jobStatus = sc.cancelled ? "failed" : jobStatuses.every(x => x === "completed") ? "completed" : jobStatuses.some(x => x === "failed") ? "failed" : "active";
const docs = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue); const fullDocs = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue);
const data = { const data = {
success: jobStatus !== "failed", success: jobStatus !== "failed",
result: { result: {
links: docs.map((doc) => { links: fullDocs.map((doc) => {
return { return {
content: doc, content: doc,
source: doc?.metadata?.sourceURL ?? doc?.url ?? "", source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
@ -242,7 +242,7 @@ async function processJob(job: Job, token: string) {
}, },
project_id: job.data.project_id, project_id: job.data.project_id,
error: message /* etc... */, error: message /* etc... */,
docs, docs: fullDocs,
}; };
await callWebhook(job.data.team_id, job.id as string, data); await callWebhook(job.data.team_id, job.id as string, data);
@ -265,6 +265,9 @@ async function processJob(job: Job, token: string) {
}); });
} }
Logger.error(error); Logger.error(error);
if (error.stack) {
Logger.error(error.stack);
}
logtail.error("Overall error ingesting", { logtail.error("Overall error ingesting", {
job_id: job.id, job_id: job.id,