From 674500affae4bf7cd044886d94abe2b5c31ee44e Mon Sep 17 00:00:00 2001
From: Nicolas <nicolascamara29@gmail.com>
Date: Tue, 4 Jun 2024 12:15:39 -0700
Subject: [PATCH 1/2] Nick:

---
 apps/api/src/lib/LLM-extraction/index.ts      |  1 -
 .../WebScraper/custom/handleCustomScraping.ts | 44 +++++++++++++++++++
 apps/api/src/scraper/WebScraper/single_url.ts | 30 ++++---------
 3 files changed, 53 insertions(+), 22 deletions(-)
 create mode 100644 apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts

diff --git a/apps/api/src/lib/LLM-extraction/index.ts b/apps/api/src/lib/LLM-extraction/index.ts
index ea6ddfdf..6614dbdf 100644
--- a/apps/api/src/lib/LLM-extraction/index.ts
+++ b/apps/api/src/lib/LLM-extraction/index.ts
@@ -1,4 +1,3 @@
-import Turndown from "turndown";
 import OpenAI from "openai";
 import Ajv from "ajv";
 const ajv = new Ajv(); // Initialize AJV for JSON schema validation
diff --git a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
new file mode 100644
index 00000000..d0f79e70
--- /dev/null
+++ b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
@@ -0,0 +1,44 @@
+export async function handleCustomScraping(
+  text: string,
+  url: string
+): Promise<{ scraper: string; url: string; wait_after_load: number } | null> {
+  // Check for Readme Docs special case
+  if (text.includes('<meta name="readme-deploy"')) {
+    console.log(
+      `Special use case detected for ${url}, using Fire Engine with wait time 1000ms`
+    );
+    return {
+      scraper: "fire-engine",
+      url: url,
+      wait_after_load: 1000,
+    };
+  }
+
+  if (text.includes('<link href="https://static.vanta.com')) {
+    console.log(
+      `Vanta link detected for ${url}, using Fire Engine with wait time 3000ms`
+    );
+    return {
+      scraper: "fire-engine",
+      url: url,
+      wait_after_load: 3000,
+    };
+  }
+
+  // Check for Google Drive PDF links in the raw HTML
+  const googleDrivePdfPattern =
+    /https:\/\/drive\.google\.com\/file\/d\/[^\/]+\/view/;
+  const googleDrivePdfLink = text.match(googleDrivePdfPattern);
+  if (googleDrivePdfLink) {
+    console.log(
+      `Google Drive PDF link detected for ${url}: ${googleDrivePdfLink[0]}`
+    );
+    return {
+      scraper: "fire-engine",
+      url: url,
+      wait_after_load: 1000,
+    };
+  }
+
+  return null;
+}
diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
index d396fad4..00c6958a 100644
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -7,6 +7,7 @@ import { parseMarkdown } from "../../lib/html-to-markdown";
 import { excludeNonMainTags } from "./utils/excludeTags";
 import { urlSpecificParams } from "./utils/custom/website_params";
 import { fetchAndProcessPdf } from "./utils/pdfProcessor";
+import { handleCustomScraping } from "./custom/handleCustomScraping";
 
 dotenv.config();
 
@@ -253,28 +254,8 @@ function getScrapingFallbackOrder(
   return scrapersInOrder as (typeof baseScrapers)[number][];
 }
 
-async function handleCustomScraping(
-  text: string,
-  url: string
-): Promise<FireEngineResponse | null> {
-  // Check for Readme Docs special case
-  if (text.includes('<meta name="readme-deploy"')) {
-    console.log(
-      `Special use case detected for ${url}, using Fire Engine with wait time 1000ms`
-    );
-    return await scrapWithFireEngine(url, 1000);
-  }
 
-  // Check for Google Drive PDF links in the raw HTML
-  const googleDrivePdfPattern = /https:\/\/drive\.google\.com\/file\/d\/[^\/]+\/view/;
-  const googleDrivePdfLink = text.match(googleDrivePdfPattern);
-  if (googleDrivePdfLink) {
-    console.log(`Google Drive PDF link detected for ${url}: ${googleDrivePdfLink[0]}`);
-    return await scrapWithFireEngine(url, 1000);
-  }
 
-  return null;
-}
 
 export async function scrapSingleUrl(
   urlToScrap: string,
@@ -345,8 +326,15 @@ export async function scrapSingleUrl(
         break;
     }
 
+    let customScrapedContent : FireEngineResponse | null = null;
+
     // Check for custom scraping conditions
-    const customScrapedContent = await handleCustomScraping(text, url);
+    const customScraperResult = await handleCustomScraping(text, url);
+
+    if(customScraperResult){
+      customScrapedContent  = await scrapWithFireEngine(customScraperResult.url, customScraperResult.wait_after_load)
+    }
+
     if (customScrapedContent) {
       text = customScrapedContent.html;
       screenshot = customScrapedContent.screenshot;

From 96257b7b17bf2364dc173db7399ecdf510053f17 Mon Sep 17 00:00:00 2001
From: Nicolas <nicolascamara29@gmail.com>
Date: Tue, 4 Jun 2024 12:22:46 -0700
Subject: [PATCH 2/2] Update handleCustomScraping.ts

---
 apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
index d0f79e70..5f6c34fe 100644
--- a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
+++ b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
@@ -14,6 +14,7 @@ export async function handleCustomScraping(
     };
   }
 
+  // Check for Vanta security portals
   if (text.includes('<link href="https://static.vanta.com')) {
     console.log(
       `Vanta link detected for ${url}, using Fire Engine with wait time 3000ms`