diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 033de6e0..2bf0dc34 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -117,6 +117,7 @@ export const scrapeOptions = z.object({ } ).transform(val => val ? val.toUpperCase() : 'US') }).optional(), + skipTlsVerification: z.boolean().default(false), }).strict(strictMessage) @@ -433,6 +434,7 @@ export function legacyScrapeOptions(x: ScrapeOptions): PageOptions { parsePDF: x.parsePDF, actions: x.actions as Action[], // no strict null checking grrrr - mogery geolocation: x.geolocation, + skipTlsVerification: x.skipTlsVerification }; } diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts index ca6142ec..8aa1d004 100644 --- a/apps/api/src/lib/entities.ts +++ b/apps/api/src/lib/entities.ts @@ -54,6 +54,7 @@ export type PageOptions = { geolocation?: { country?: string; }; + skipTlsVerification?: boolean; }; export type ExtractorOptions = { diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index 5285a9f4..1817a07b 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -594,6 +594,7 @@ export class WebScraperDataProvider { atsv: options.pageOptions?.atsv ?? false, actions: options.pageOptions?.actions ?? undefined, geolocation: options.pageOptions?.geolocation ?? undefined, + skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false, }; this.extractorOptions = options.extractorOptions ?? { mode: "markdown" }; this.replaceAllPathsWithAbsolutePaths = diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index d1dafd5f..3bbd74eb 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -28,7 +28,7 @@ export async function scrapWithFireEngine({ waitFor = 0, screenshot = false, fullPageScreenshot = false, - pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" } }, + pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false }, fireEngineOptions = {}, headers, options, @@ -40,7 +40,7 @@ export async function scrapWithFireEngine({ waitFor?: number; screenshot?: boolean; fullPageScreenshot?: boolean; - pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string } }; + pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean }; fireEngineOptions?: FireEngineOptions; headers?: Record; options?: any; @@ -119,6 +119,7 @@ export async function scrapWithFireEngine({ atsv: pageOptions?.atsv ?? false, scrollXPaths: pageOptions?.scrollXPaths ?? [], geolocation: pageOptions?.geolocation, + skipTlsVerification: pageOptions?.skipTlsVerification ?? false, actions: actions, }, { diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index b4141dc1..cd76793c 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -157,6 +157,7 @@ export async function scrapSingleUrl( atsv: pageOptions.atsv ?? false, actions: pageOptions.actions ?? undefined, geolocation: pageOptions.geolocation ?? undefined, + skipTlsVerification: pageOptions.skipTlsVerification ?? false, } if (extractorOptions) {