Merge pull request #901 from mendableai/nsc/sitemap-only

Allows `/map` to only return links present in the sitemap

apps/api/src/controllers/v1/map.ts

@@ -1,10 +1,6 @@
 import { Response } from "express";
 import { v4 as uuidv4 } from "uuid";
-import {
-  mapRequestSchema,
-  RequestWithAuth,
-  scrapeOptions,
-} from "./types";
+import { mapRequestSchema, RequestWithAuth, scrapeOptions } from "./types";
 import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
 import { MapResponse, MapRequest } from "./types";
 import { configDotenv } from "dotenv";
@@ -46,6 +42,7 @@ export async function mapController(
     originUrl: req.body.url,
     crawlerOptions: {
       ...req.body,
+      limit: req.body.sitemapOnly ? 10000000 : limit,
       scrapeOptions: undefined,
     },
     scrapeOptions: scrapeOptions.parse({}),
@@ -57,6 +54,15 @@ export async function mapController(
 
   const crawler = crawlToCrawler(id, sc);
 
+  // If sitemapOnly is true, only get links from sitemap
+  if (req.body.sitemapOnly) {
+    const sitemap = await crawler.tryGetSitemap();
+    if (sitemap !== null) {
+      sitemap.forEach((x) => {
+        links.push(x.url);
+      });
+    }
+  } else {
     let urlWithoutWww = req.body.url.replace("www.", "");
 
     let mapUrl = req.body.search
@@ -64,7 +70,9 @@ export async function mapController(
       : `site:${req.body.url}`;
 
     const resultsPerPage = 100;
-  const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage);
+    const maxPages = Math.ceil(
+      Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage
+    );
 
     const cacheKey = `fireEngineMap:${mapUrl}`;
     const cachedResult = null;
@@ -82,7 +90,9 @@ export async function mapController(
         });
       };
 
-    pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1));
+      pagePromises = Array.from({ length: maxPages }, (_, i) =>
+        fetchPage(i + 1)
+      );
       allResults = await Promise.all(pagePromises);
 
       await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours
@@ -128,6 +138,8 @@ export async function mapController(
       }
     }
 
+    
+  }
   // Perform cosine similarity between the search query and the list of links
   if (req.body.search) {
     const searchQuery = req.body.search.toLowerCase();

apps/api/src/controllers/v1/types.ts

@@ -261,6 +261,7 @@ export const mapRequestSchema = crawlerOptions.extend({
   includeSubdomains: z.boolean().default(true),
   search: z.string().optional(),
   ignoreSitemap: z.boolean().default(false),
+  sitemapOnly: z.boolean().default(false),
   limit: z.number().min(1).max(5000).default(5000),
 }).strict(strictMessage);
 

apps/api/src/search/fireEngine.ts

@@ -1,4 +1,3 @@
-import axios from "axios";
 import dotenv from "dotenv";
 import { SearchResult } from "../../src/lib/entities";
 import * as Sentry from "@sentry/node";
@@ -6,7 +5,6 @@ import { logger } from "../lib/logger";
 
 dotenv.config();
 
-
 export async function fireEngineMap(
   q: string,
   options: {
@@ -37,18 +35,18 @@ export async function fireEngineMap(
       return [];
     }
 
-    let config = {
+    const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/search`, {
       method: "POST",
-      url: `${process.env.FIRE_ENGINE_BETA_URL}/search`,
       headers: {
         "Content-Type": "application/json",
-        "X-Disable-Cache": "true"
+        "X-Disable-Cache": "true",
       },
-      data: data,
-    };
-    const response = await axios(config);
-    if (response && response.data) {
-      return response.data;
+      body: data,
+    });
+
+    if (response.ok) {
+      const responseData = await response.json();
+      return responseData;
     } else {
       return [];
     }

apps/js-sdk/firecrawl/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "1.8.3",
+  "version": "1.8.4",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

apps/js-sdk/firecrawl/src/index.ts

@@ -221,6 +221,7 @@ export interface MapParams {
   search?: string;
   ignoreSitemap?: boolean;
   includeSubdomains?: boolean;
+  sitemapOnly?: boolean;
   limit?: number;
 }