Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(scrapeURL): reduce logs

Browse Source
This commit is contained in:
Gergő Móricz 2024-11-11 20:51:29 +01:00
parent 219f4732a0
commit d13a2e7d26
3 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
View File

@ -87,7 +87,6 @@ export async function fireEngineCheckStatus(logger: Logger, jobId: string): Prom
logger.debug("Scrape succeeded!", { jobId }); logger.debug("Scrape succeeded!", { jobId });
return successParse.data; return successParse.data;
} else if (processingParse.success) { } else if (processingParse.success) {
logger.debug("Scrape is still processing", { jobId });
throw new StillProcessingError(jobId); throw new StillProcessingError(jobId);
} else if (failedParse.success) { } else if (failedParse.success) {
logger.debug("Scrape job failed", { status, jobId }); logger.debug("Scrape job failed", { status, jobId });

2
apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
View File

@ -40,7 +40,7 @@ async function performFireEngineScrape<Engine extends FireEngineScrapeRequestChr
status = await fireEngineCheckStatus(logger.child({ method: "fireEngineCheckStatus" }), scrape.jobId) status = await fireEngineCheckStatus(logger.child({ method: "fireEngineCheckStatus" }), scrape.jobId)
} catch (error) { } catch (error) {
if (error instanceof StillProcessingError) { if (error instanceof StillProcessingError) {
logger.debug("Scrape is still processing..."); // nop
} else if (error instanceof EngineError) { } else if (error instanceof EngineError) {
logger.debug("Fire-engine scrape job failed.", { error, jobId: scrape.jobId }); logger.debug("Fire-engine scrape job failed.", { error, jobId: scrape.jobId });
throw error; throw error;

7
apps/api/src/scraper/scrapeURL/transformers/index.ts
View File

@ -115,16 +115,19 @@ export const transformerStack: Transformer[] = [
]; ];
export async function executeTransformers(meta: Meta, document: Document): Promise<Document> { export async function executeTransformers(meta: Meta, document: Document): Promise<Document> {
const executions: [string, number][] = [];
for (const transformer of transformerStack) { for (const transformer of transformerStack) {
const _meta = { const _meta = {
...meta, ...meta,
logger: meta.logger.child({ method: "executeTransformers/" + transformer.name }), logger: meta.logger.child({ method: "executeTransformers/" + transformer.name }),
}; };
meta.logger.debug("Executing transformer " + transformer.name + "...", { document });
const start = Date.now(); const start = Date.now();
document = await transformer(_meta, document); document = await transformer(_meta, document);
meta.logger.debug("Finished executing transformer " + transformer.name + " (" + (Date.now() - start) + "ms)", { document }); executions.push([transformer.name, Date.now() - start]);
} }
meta.logger.debug("Executed transformers.", { executions });
return document; return document;
} }