Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-15 19:22:19 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(crawl-redis): batch scrape lockURL
Some checks are pending
Deploy Images to GHCR / push-app-image (push) Waiting to run
Details

Browse Source
This commit is contained in:
Móricz Gergő 2024-11-12 11:52:34 +01:00
parent d8bb1f68c6
commit f6db9f1428
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/lib/crawl-redis.ts
View File

@ -144,7 +144,7 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise
await redisConnection.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60, "NX"); await redisConnection.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60, "NX");
let res: boolean; let res: boolean;
if (!sc.crawlerOptions.deduplicateSimilarURLs) { if (!sc.crawlerOptions?.deduplicateSimilarURLs) {
res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0 res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0
} else { } else {
const permutations = generateURLPermutations(url); const permutations = generateURLPermutations(url);