Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Update index.ts

Browse Source
This commit is contained in:
Nicolas 2024-09-09 12:26:55 -03:00
parent 60a15d00eb
commit 22a5e85899
1 changed files with 68 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
apps/api/src/index.ts
View File

@ -1,5 +1,5 @@
import "dotenv/config";
import "./services/sentry"
import "./services/sentry";
import * as Sentry from "@sentry/node";
import express, { NextFunction, Request, Response } from "express";
import bodyParser from "body-parser";
@ -12,9 +12,9 @@ import os from "os";
import { Logger } from "./lib/logger";
import { adminRouter } from "./routes/admin";
import { ScrapeEvents } from "./lib/scrape-events";
import http from 'node:http';
import https from 'node:https';
import CacheableLookup from 'cacheable-lookup';
import http from "node:http";
import https from "node:https";
import CacheableLookup from "cacheable-lookup";
import { v1Router } from "./routes/v1";
import expressWs from "express-ws";
import { crawlStatusWSController } from "./controllers/v1/crawl-status-ws";
@ -31,11 +31,11 @@ Logger.info(`Number of CPUs: ${numCPUs} available`);
const cacheable = new CacheableLookup({
// this is important to avoid querying local hostnames see https://github.com/szmarczak/cacheable-lookup readme
lookup:false
lookup: false,
});
cacheable.install(http.globalAgent);
cacheable.install(https.globalAgent)
cacheable.install(https.globalAgent);
if (cluster.isMaster) {
Logger.info(`Master ${process.pid} is running`);
@ -115,9 +115,7 @@ if (cluster.isMaster) {
app.get(`/serverHealthCheck`, async (req, res) => {
try {
const scrapeQueue = getScrapeQueue();
const [waitingJobs] = await Promise.all([
scrapeQueue.getWaitingCount(),
]);
const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);
const noWaitingJobs = waitingJobs === 0;
// 200 if no active jobs, 503 if there are active jobs
@ -190,17 +188,43 @@ if (cluster.isMaster) {
res.send({ isProduction: global.isProduction });
});
app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response<ErrorResponse>, next: NextFunction) => {
app.use(
(
err: unknown,
req: Request<{}, ErrorResponse, undefined>,
res: Response<ErrorResponse>,
next: NextFunction
) => {
if (err instanceof ZodError) {
res.status(400).json({ success: false, error: "Bad Request", details: err.errors });
res
.status(400)
.json({ success: false, error: "Bad Request", details: err.errors });
} else {
next(err);
}
});
}
);
Sentry.setupExpressErrorHandler(app);
app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: ResponseWithSentry<ErrorResponse>, next: NextFunction) => {
app.use(
(
err: unknown,
req: Request<{}, ErrorResponse, undefined>,
res: ResponseWithSentry<ErrorResponse>,
next: NextFunction
) => {
if (
err instanceof SyntaxError &&
"status" in err &&
err.status === 400 &&
"body" in err
) {
return res
.status(400)
.json({ success: false, error: "Bad request, malformed JSON" });
}
const id = res.sentry ?? uuidv4();
let verbose = JSON.stringify(err);
if (verbose === "{}") {
@ -213,15 +237,28 @@ if (cluster.isMaster) {
}
}
Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose);
res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + id });
Logger.error(
"Error occurred in request! (" +
req.path +
") -- ID " +
id +
" -- " +
verbose
);
res
.status(500)
.json({
success: false,
error:
"An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " +
id,
});
}
);
Logger.info(`Worker ${process.pid} started`);
}
// const sq = getScrapeQueue();
// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));
@ -230,6 +267,3 @@ if (cluster.isMaster) {
// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));
// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));
// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));