From f6db9f1428609e1debe2903464286ad6c1ec2489 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?M=C3=B3ricz=20Gerg=C5=91?= <mo.geryy@gmail.com>
Date: Tue, 12 Nov 2024 11:52:34 +0100
Subject: [PATCH] fix(crawl-redis): batch scrape lockURL

---
 apps/api/src/lib/crawl-redis.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 9f6a8cf4..b5936ad6 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -144,7 +144,7 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise
     await redisConnection.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60, "NX");
 
     let res: boolean;
-    if (!sc.crawlerOptions.deduplicateSimilarURLs) {
+    if (!sc.crawlerOptions?.deduplicateSimilarURLs) {
         res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0
     } else {
         const permutations = generateURLPermutations(url);