Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Update index.test.ts

Browse Source
This commit is contained in:
Nicolas 2024-05-08 12:26:04 -07:00
parent b7e3104c7b
commit a0a67f124a
1 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/test-suite/index.test.ts
View File

@ -57,7 +57,7 @@ describe("Scraping/Crawling Checkup (E2E)", () => {
.post("/v0/scrape")
.set("Content-Type", "application/json")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.send({ url: websiteData.website });
.send({ url: websiteData.website, pageOptions: { onlyMainContent: true } });
if (scrapedContent.statusCode !== 200) {
console.error(`Failed to scrape ${websiteData.website}`);
@ -138,15 +138,14 @@ describe("Scraping/Crawling Checkup (E2E)", () => {
batchPromises.push(batchPromise);
}
const responses = (await Promise.all(batchPromises)).flat();
const validResponses = responses.filter((response) => response !== null);
const score = (passedTests / validResponses.length) * 100;
(await Promise.all(batchPromises)).flat();
const score = (passedTests / websitesData.length) * 100;
const endTime = new Date().getTime();
const timeTaken = (endTime - startTime) / 1000;
console.log(`Score: ${score}%`);
console.log(`Total tokens: ${totalTokens}`);
await logErrors(errorLog, timeTaken, totalTokens, score, validResponses.length);
await logErrors(errorLog, timeTaken, totalTokens, score, websitesData.length);
if (process.env.ENV === "local" && errorLog.length > 0) {
if (!fs.existsSync(logsDir)){