From 503e83e83edcf578ab7acec11275ad5695bc3c41 Mon Sep 17 00:00:00 2001
From: Sebastjan Prachovskij <sebastjan.prachovskij@gmail.com>
Date: Tue, 3 Sep 2024 18:26:11 +0300
Subject: [PATCH] Add SearchApi to search

Add support for engines, improve status code error

Remove changes in package, add engine to env params

Improve description in env example

Remove unnecessary empty line

Improve text
---
 apps/api/.env.example            | 11 ++++--
 apps/api/.env.local              |  2 +-
 apps/api/src/search/index.ts     | 12 ++++++-
 apps/api/src/search/searchapi.ts | 60 ++++++++++++++++++++++++++++++++
 4 files changed, 80 insertions(+), 5 deletions(-)
 create mode 100644 apps/api/src/search/searchapi.ts

diff --git a/apps/api/.env.example b/apps/api/.env.example
index f3c1dc1b..6ba49daa 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -1,5 +1,5 @@
 # ===== Required ENVS ======
-NUM_WORKERS_PER_QUEUE=8 
+NUM_WORKERS_PER_QUEUE=8
 PORT=3002
 HOST=0.0.0.0
 REDIS_URL=redis://redis:6379 #for self-hosting using docker, use redis://redis:6379. For running locally, use redis://localhost:6379
@@ -11,9 +11,14 @@ USE_DB_AUTHENTICATION=true
 
 # ===== Optional ENVS ======
 
+# SearchApi key. Head to https://searchapi.com/ to get your API key
+SEARCHAPI_API_KEY=
+# SearchApi engine, defaults to google. Available options: google, bing, baidu, google_news, etc. Head to https://searchapi.com/ to explore more engines
+SEARCHAPI_ENGINE=
+
 # Supabase Setup (used to support DB authentication, advanced logging, etc.)
-SUPABASE_ANON_TOKEN= 
-SUPABASE_URL= 
+SUPABASE_ANON_TOKEN=
+SUPABASE_URL=
 SUPABASE_SERVICE_TOKEN=
 
 # Other Optionals
diff --git a/apps/api/.env.local b/apps/api/.env.local
index 17f85935..9fa41498 100644
--- a/apps/api/.env.local
+++ b/apps/api/.env.local
@@ -12,4 +12,4 @@ ANTHROPIC_API_KEY=
 BULL_AUTH_KEY=
 LOGTAIL_KEY=
 PLAYWRIGHT_MICROSERVICE_URL=
-
+SEARCHAPI_API_KEY=
diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.ts
index f4c5b6d0..3bcb85d2 100644
--- a/apps/api/src/search/index.ts
+++ b/apps/api/src/search/index.ts
@@ -2,6 +2,7 @@ import { Logger } from "../../src/lib/logger";
 import { SearchResult } from "../../src/lib/entities";
 import { googleSearch } from "./googlesearch";
 import { fireEngineMap } from "./fireEngine";
+import { searchapi_search } from "./searchapi";
 import { serper_search } from "./serper";
 
 export async function search({
@@ -30,7 +31,16 @@ export async function search({
   timeout?: number;
 }): Promise<SearchResult[]> {
   try {
-    
+    if (process.env.SEARCHAPI_API_KEY) {
+      return await searchapi_search(query, {
+        num_results,
+        tbs,
+        filter,
+        lang,
+        country,
+        location
+      });
+    }
     if (process.env.SERPER_API_KEY) {
       return await serper_search(query, {
         num_results,
diff --git a/apps/api/src/search/searchapi.ts b/apps/api/src/search/searchapi.ts
new file mode 100644
index 00000000..24778a77
--- /dev/null
+++ b/apps/api/src/search/searchapi.ts
@@ -0,0 +1,60 @@
+import axios from "axios";
+import dotenv from "dotenv";
+import { SearchResult } from "../../src/lib/entities";
+
+dotenv.config();
+
+interface SearchOptions {
+  tbs?: string;
+  filter?: string;
+  lang?: string;
+  country?: string;
+  location?: string;
+  num_results: number;
+  page?: number;
+}
+
+export async function searchapi_search(q: string, options: SearchOptions): Promise<SearchResult[]> {
+  const params = {
+    q: q,
+    hl: options.lang,
+    gl: options.country,
+    location: options.location,
+    num: options.num_results,
+    page: options.page ?? 1,
+    engine: process.env.SEARCHAPI_ENGINE || "google",
+  };
+
+  const url = `https://www.searchapi.io/api/v1/search`;
+
+  try {
+    const response = await axios.get(url, {
+      headers: {
+        "Authorization": `Bearer ${process.env.SEARCHAPI_API_KEY}`,
+        "Content-Type": "application/json",
+        "X-SearchApi-Source": "Firecrawl",
+      },
+      params: params,
+    });
+
+
+    if (response.status === 401) {
+      throw new Error("Unauthorized. Please check your API key.");
+    }
+
+    const data = response.data;
+
+    if (data && Array.isArray(data.organic_results)) {
+      return data.organic_results.map((a: any) => ({
+        url: a.link,
+        title: a.title,
+        description: a.snippet,
+      }));
+    } else {
+      return [];
+    }
+  } catch (error) {
+    console.error(`There was an error searching for content: ${error.message}`);
+    return [];
+  }
+}