Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-09-01 13:52:34 -03:00
parent 979697df1c
commit b68a50fea3
2 changed files with 41 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
apps/api/src/services/queue-worker.ts
View File

@ -221,12 +221,10 @@ async function processJob(job: Job, token: string) {
// No idea what this does and when it is called.
if (job.data.mode === "crawl" && !job.data.v1) {
callWebhook(job.data.team_id, job.id as string, data, job.data.webhook, job.data.v1).catch((error) => {
Logger.error(`Error calling webhook for job (1 - mode crawl - v0) ${job.id} - ${error}`);
});
callWebhook(job.data.team_id, job.id as string, data, job.data.webhook, job.data.v1);
}
if (job.data.webhook && job.data.mode !== "crawl" && job.data.v1) {
await callWebhook(job.data.team_id, job.data.crawl_id, data, job.data.webhook, job.data.v1);
callWebhook(job.data.team_id, job.data.crawl_id, data, job.data.webhook, job.data.v1);
}
if (job.data.crawl_id) {
@ -351,15 +349,11 @@ async function processJob(job: Job, token: string) {
};
// v0 web hooks, call when done with all the data
if (!job.data.v1) {
callWebhook(job.data.team_id, job.data.crawl_id, data, job.data.webhook, job.data.v1, "crawl.completed").catch((error) => {
Logger.error(`Error calling webhook for job ${job.id} - ${error}`);
});
callWebhook(job.data.team_id, job.data.crawl_id, data, job.data.webhook, job.data.v1, "crawl.completed");
}
// v1 web hooks, call when done with no data, but with event completed
if (job.data.v1 && job.data.webhook) {
callWebhook(job.data.team_id, job.data.crawl_id, [], job.data.webhook, job.data.v1, "crawl.completed").catch((error) => {
Logger.error(`Error calling webhook for job ${job.id} - ${error}`);
});
callWebhook(job.data.team_id, job.data.crawl_id, [], job.data.webhook, job.data.v1, "crawl.completed");
}
}
}
@ -407,14 +401,10 @@ async function processJob(job: Job, token: string) {
};
if (!job.data.v1 && (job.data.mode === "crawl" || job.data.crawl_id)) {
callWebhook(job.data.team_id, job.data.crawl_id ?? job.id as string, data, job.data.webhook, job.data.v1).catch((error) => {
Logger.error(`Error calling webhook for job (catch - v0) ${job.id} - ${error}`);
});
callWebhook(job.data.team_id, job.data.crawl_id ?? job.id as string, data, job.data.webhook, job.data.v1);
}
if(job.data.v1) {
callWebhook(job.data.team_id, job.id as string, [], job.data.webhook, job.data.v1, "crawl.failed").catch((error) => {
Logger.error(`Error calling webhook for job (catch - v1) ${job.id} - ${error}`);
});
callWebhook(job.data.team_id, job.id as string, [], job.data.webhook, job.data.v1, "crawl.failed");
}
if (job.data.crawl_id) {

55
apps/api/src/services/webhook.ts
View File

@ -43,7 +43,12 @@ export const callWebhook = async (
}
let dataToSend = [];
if (data && data.result && data.result.links && data.result.links.length !== 0) {
if (
data &&
data.result &&
data.result.links &&
data.result.links.length !== 0
) {
for (let i = 0; i < data.result.links.length; i++) {
if (v1) {
dataToSend.push(
@ -59,26 +64,36 @@ export const callWebhook = async (
}
}
axios.post(
webhookUrl,
{
success: !v1 ? data.success : eventType === "crawl.page" ? data.success : true,
type: eventType,
[v1 ? 'id' : 'jobId']: id,
data: dataToSend,
error: !v1 ? data?.error || undefined : eventType === "crawl.page" ? data?.error || undefined : undefined,
},
{
headers: {
"Content-Type": "application/json",
axios
.post(
webhookUrl,
{
success: !v1
? data.success
: eventType === "crawl.page"
? data.success
: true,
type: eventType,
[v1 ? "id" : "jobId"]: id,
data: dataToSend,
error: !v1
? data?.error || undefined
: eventType === "crawl.page"
? data?.error || undefined
: undefined,
},
timeout: 10000, // 10 seconds timeout
}
).catch((error) => {
Logger.error(
`Error sending webhook for team ID: ${teamId}, error: ${error.message}`
);
});
{
headers: {
"Content-Type": "application/json",
},
timeout: 10000, // 10 seconds timeout
}
)
.catch((error) => {
Logger.error(
`Error sending webhook for team ID: ${teamId}, error: ${error.message}`
);
});
} catch (error) {
Logger.debug(
`Error sending webhook for team ID: ${teamId}, error: ${error.message}`