Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

move param to the right place

Browse Source
This commit is contained in:
Gergő Móricz 2024-11-08 16:25:11 +01:00
parent 6ecf24b85e
commit dc3a4e27fd
3 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/controllers/v1/types.ts
View File

@ -143,7 +143,6 @@ export const scrapeOptions = z.object({
}).optional(), }).optional(),
skipTlsVerification: z.boolean().default(false), skipTlsVerification: z.boolean().default(false),
removeBase64Images: z.boolean().default(true), removeBase64Images: z.boolean().default(true),
deduplicateSimilarURLs: z.boolean().default(true),
}).strict(strictMessage) }).strict(strictMessage)
@ -200,6 +199,7 @@ const crawlerOptions = z.object({
allowBackwardLinks: z.boolean().default(false), // >> TODO: CHANGE THIS NAME??? allowBackwardLinks: z.boolean().default(false), // >> TODO: CHANGE THIS NAME???
allowExternalLinks: z.boolean().default(false), allowExternalLinks: z.boolean().default(false),
ignoreSitemap: z.boolean().default(true), ignoreSitemap: z.boolean().default(true),
deduplicateSimilarURLs: z.boolean().default(true),
}).strict(strictMessage); }).strict(strictMessage);
// export type CrawlerOptions = { // export type CrawlerOptions = {

2
apps/api/src/lib/crawl-redis.ts
View File

@ -107,7 +107,7 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise
url = normalizeURL(url); url = normalizeURL(url);
let res: boolean; let res: boolean;
if (!sc.scrapeOptions.deduplicateSimilarURLs) { if (!sc.crawlerOptions.deduplicateSimilarURLs) {
res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0 res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0
} else { } else {
const urlO = new URL(url); const urlO = new URL(url);

2
apps/js-sdk/firecrawl/src/index.ts
View File

@ -88,7 +88,6 @@ export interface CrawlScrapeOptions {
}; };
skipTlsVerification?: boolean; skipTlsVerification?: boolean;
removeBase64Images?: boolean; removeBase64Images?: boolean;
deduplicateSimilarURLs?: boolean;
} }
export type Action = { export type Action = {
@ -151,6 +150,7 @@ export interface CrawlParams {
ignoreSitemap?: boolean; ignoreSitemap?: boolean;
scrapeOptions?: CrawlScrapeOptions; scrapeOptions?: CrawlScrapeOptions;
webhook?: string; webhook?: string;
deduplicateSimilarURLs?: boolean;
} }
/** /**