feat: 可选的对文献相关性检验

app/i18n/locales/en/translation.json

@@ -27,5 +27,6 @@
     "linuxdo": "linuxdo(Recommended)",
     "custom": "Custom"
   },
-  "鼠标点击段落中的上标跳转到文献引用？": "Click the superscript in the paragraph to jump to the reference?"
+  "鼠标点击段落中的上标跳转到文献引用？": "Click the superscript in the paragraph to jump to the reference?",
+  "是否检查文献与主题相关性(如果不相关则不会传给AI引用)": "Check the relevance of the literature to the topic (if it is not relevant, it will not be passed to the AI reference)"
 }

app/i18n/locales/zh-CN/translation.json

@@ -28,5 +28,6 @@
     "linuxdo": "linuxdo(第二个推荐)",
     "custom": "自定义"
   },
-  "鼠标点击段落中的上标跳转到文献引用？": "鼠标点击段落中的上标跳转到文献引用？"
+  "鼠标点击段落中的上标跳转到文献引用？": "鼠标点击段落中的上标跳转到文献引用？",
+  "是否检查文献与主题相关性(如果不相关则不会传给AI引用)": "是否检查文献与主题相关性（如果不相关则不会传给AI引用）"
 }

app/store/index.ts

@@ -29,6 +29,7 @@ const statePersistConfig = {
     "isVip",
     "language",
     "isJumpToReference",
+    "isEvaluateTopicMatch",
   ],
 };
 

app/store/slices/stateSlice.ts

@@ -6,6 +6,7 @@ export interface APIState {
   isVip: boolean;
   language: string;
   isJumpToReference: boolean;
+  isEvaluateTopicMatch: boolean;
 }
 
 const initialState: APIState = {
@@ -15,6 +16,7 @@ const initialState: APIState = {
   isVip: false,
   language: "en",
   isJumpToReference: false,
+  isEvaluateTopicMatch: false,
 };
 
 export const stateSlice = createSlice({
@@ -45,6 +47,9 @@ export const stateSlice = createSlice({
     setIsJumpToReference: (state, action: PayloadAction<boolean>) => {
       state.isJumpToReference = action.payload;
     },
+    setIsEvaluateTopicMatch: (state, action: PayloadAction<boolean>) => {
+      state.isEvaluateTopicMatch = action.payload;
+    },
   },
 });
 
@@ -56,6 +61,7 @@ export const {
   setIsVip,
   setLanguage,
   setIsJumpToReference,
+  setIsEvaluateTopicMatch,
 } = stateSlice.actions;
 
 export const stateReducer = stateSlice.reducer;

components/GetSemantic.tsx

@@ -32,7 +32,7 @@ async function getSemanticPapers(
       params: {
         query: query,
         offset: offset,
-        limit: 2,
+        limit: limit,
         year: year,
         fields: "title,year,authors.name,abstract,venue,url,journal",
       },

components/QuillEditor.tsx

@@ -10,7 +10,7 @@ import Link from "next/link";
 import getArxivPapers from "./GetArxiv";
 import getSemanticPapers from "./GetSemantic";
 import { fetchPubMedData } from "./GetPubMed ";
-import { getTopicFromAI, sendMessageToOpenAI } from "./chatAI";
+import { getAI, sendMessageToOpenAI } from "./chatAI";
 import {
   getTextBeforeCursor,
   convertToSuperscript,
@@ -19,6 +19,7 @@ import {
   getNumberBeforeCursor,
   formatJournalReference,
 } from "@/utils/others/quillutils";
+import { evaluateTopicMatch } from "@/utils/others/aiutils";
 //组件
 import ExportDocx from "./Export";
 import ReferenceList from "./ReferenceList";
@@ -74,6 +75,9 @@ const QEditor = ({ lng }) => {
   const isJumpToReference = useAppSelector(
     (state) => state.state.isJumpToReference
   );
+  const isEvaluateTopicMatch = useAppSelector(
+    (state) => state.state.isEvaluateTopicMatch
+  );
   const [quill, setQuill] = useState<Quill | null>(null);
   const contentUpdatedFromNetwork = useAppSelector(
     (state) => state.state.contentUpdatedFromNetwork
@@ -269,8 +273,14 @@ const QEditor = ({ lng }) => {
   async function paper2AI(topic: string) {
     quill!.setSelection(cursorPosition!, 0); // 将光标移动到原来的位置
     let offset = -1;
-    if (generatedPaperNumber) offset = 0;
+    if (generatedPaperNumber != 1) offset = 0; //如果生成的数量不为1，则从0开始
-    setOpenProgressBar(true);
+    setOpenProgressBar(true); //开启进度条
+    //如果说要评估主题是否匹配的话,就要多获取一些文献
+    let limit = 2;
+    if (isEvaluateTopicMatch) {
+      limit = 4;
+    }
+
     for (let i = 0; i < generatedPaperNumber!; i++) {
       try {
         if (!topic) {
@@ -278,7 +288,13 @@ const QEditor = ({ lng }) => {
           const prompt =
             "As a topic extraction assistant, you can help me extract the current discussion of the paper topic, I will enter the content of the paper, you extract the paper topic , no more than two, Hyphenated query terms yield no matches (replace it with space to find matches) return format is: topic1 topic2";
           const userMessage = getTextBeforeCursor(quill!, 2000);
-          topic = await getTopicFromAI(userMessage, prompt, apiKey);
+          topic = await getAI(
+            userMessage,
+            prompt,
+            apiKey,
+            upsreamUrl,
+            selectedModel!
+          );
           console.log("topic in AI before removeSpecialCharacters", topic);
           topic = removeSpecialCharacters(topic);
           topic = topic.split(" ").slice(0, 2).join(" ");
@@ -290,7 +306,19 @@ const QEditor = ({ lng }) => {
         console.log("topic in AI", topic);
         let rawData, dataString, newReferences;
         if (selectedSource === "arxiv") {
-          rawData = await getArxivPapers(topic);
+          rawData = await getArxivPapers(topic, limit, offset);
+          //判断返回的文献是否跟用户输入的主题相关
+          if (isEvaluateTopicMatch) {
+            const { relevantPapers, nonRelevantPapers } =
+              await evaluateTopicMatch(
+                rawData,
+                apiKey,
+                upsreamUrl,
+                selectedModel!,
+                topic
+              );
+            rawData = relevantPapers;
+          }
           console.log("arxiv rawdata:", rawData);
           // 将 rawData 转换为引用数组
           newReferences = rawData.map((entry: any) => ({
@@ -305,7 +333,19 @@ const QEditor = ({ lng }) => {
             })
             .join("");
         } else if (selectedSource === "semanticScholar") {
-          rawData = await getSemanticPapers(topic, "2015-2023", offset);
+          rawData = await getSemanticPapers(topic, "2015-2023", offset, limit);
+          //判断返回的文献是否跟用户输入的主题相关
+          if (isEvaluateTopicMatch) {
+            const { relevantPapers, nonRelevantPapers } =
+              await evaluateTopicMatch(
+                rawData,
+                apiKey,
+                upsreamUrl,
+                selectedModel!,
+                topic
+              );
+            rawData = relevantPapers;
+          }
           // 将 rawData 转换为引用数组
           newReferences = rawData.map((entry: any) => ({
             url: entry.url,
@@ -321,10 +361,22 @@ const QEditor = ({ lng }) => {
             })
             .join("");
         } else if (selectedSource === "pubmed") {
-          rawData = await fetchPubMedData(topic, 2020, offset, 2);
+          rawData = await fetchPubMedData(topic, 2020, offset, limit);
           if (!rawData) {
             throw new Error("未搜索到文献 from PubMed.");
           }
+          //判断返回的文献是否跟用户输入的主题相关
+          if (isEvaluateTopicMatch) {
+            const { relevantPapers, nonRelevantPapers } =
+              await evaluateTopicMatch(
+                rawData,
+                apiKey,
+                upsreamUrl,
+                selectedModel!,
+                topic
+              );
+            rawData = relevantPapers;
+          }
           newReferences = rawData.map((entry: any) => ({
             id: entry.id, // 文章的 PubMed ID
             title: entry.title, // 文章的标题
@@ -336,9 +388,8 @@ const QEditor = ({ lng }) => {
             source: "PubMed", // 指示这些引用来自 PubMed
           }));
 
-          // 打印或进一步处理 newReferences
+          // 打印 newReferences
           console.log(newReferences);
-
           dataString = rawData
             .map((entry: any) => {
               return `Time: ${entry.year}\nTitle: ${entry.title}\nSummary: ${entry.abstract}\n\n`;

components/Settings.tsx

@@ -7,7 +7,10 @@ import {
   setUpsreamUrl,
   setSystemPrompt,
 } from "@/app/store/slices/authSlice";
-import { setIsJumpToReference } from "@/app/store/slices/stateSlice";
+import {
+  setIsJumpToReference,
+  setIsEvaluateTopicMatch,
+} from "@/app/store/slices/stateSlice";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { faArrowLeft } from "@fortawesome/free-solid-svg-icons";
 import Link from "next/link";
@@ -53,14 +56,16 @@ const Settings = ({ lng }: { lng: string }) => {
   const isJumpToReference = useAppSelector(
     (state) => state.state.isJumpToReference
   );
+  const isEvaluateTopicMatch = useAppSelector(
+    (state) => state.state.isEvaluateTopicMatch
+  );
   //state
   const [userConfigNumber, setUserConfigNumber] = useLocalStorage(
     "userConfigNumber",
     "2"
   );
-
+  const toggleSwitch = (currentState: any, setState: any) => {
-  const toggleSwitch = () => {
+    setState(!currentState);
-    dispatch(setIsJumpToReference(!isJumpToReference));
   };
   return (
     <div className="max-w-md rounded overflow-hidden shadow-lg bg-blue-gray-100 z-1000  mx-auto ">
@@ -153,7 +158,11 @@ const Settings = ({ lng }: { lng: string }) => {
           type="checkbox"
           className="sr-only peer"
           checked={isJumpToReference}
-          onChange={toggleSwitch}
+          onChange={() =>
+            toggleSwitch(isJumpToReference, (value: any) =>
+              dispatch(setIsJumpToReference(value))
+            )
+          }
         />
         <div className="w-10 h-4 bg-gray-200 rounded-full peer-checked:bg-blue-600 peer-focus:ring-4 peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800 transition-colors ease-in-out duration-200"></div>
         <span
@@ -163,6 +172,25 @@ const Settings = ({ lng }: { lng: string }) => {
         ></span>
         {t("鼠标点击段落中的上标跳转到文献引用？")}
       </label>
+      <label className="relative inline-flex items-center cursor-pointer">
+        <input
+          type="checkbox"
+          className="sr-only peer"
+          checked={isEvaluateTopicMatch}
+          onChange={() =>
+            toggleSwitch(isEvaluateTopicMatch, (value: any) =>
+              dispatch(setIsEvaluateTopicMatch(value))
+            )
+          }
+        />
+        <div className="w-10 h-4 bg-gray-200 rounded-full peer-checked:bg-blue-600 peer-focus:ring-4 peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800 transition-colors ease-in-out duration-200"></div>
+        <span
+          className={`absolute block bg-white w-3 h-3 rounded-full transition ease-in-out duration-200 transform ${
+            isJumpToReference ? "translate-x-6" : "translate-x-1"
+          } -translate-y-1/2 top-1/2`}
+        ></span>
+        {t("是否检查文献与主题相关性(如果不相关则不会传给AI引用)")}
+      </label>
     </div>
   );
 };

components/chatAI.tsx

@@ -105,10 +105,12 @@ const sendMessageToOpenAI = async (
   }
 };
 
-const getTopicFromAI = async (
+const getAI = async (
   userMessage: string,
-  prompt: string,
+  systemPrompt: string,
-  apiKey: string
+  apiKey: string,
+  upsreamUrl: string,
+  selectedModel: string
 ) => {
   // 设置API请求参数
   const requestOptions = {
@@ -122,12 +124,12 @@ const getTopicFromAI = async (
           : process.env.NEXT_PUBLIC_OPENAI_API_KEY),
     },
     body: JSON.stringify({
-      model: "gpt-3.5-turbo",
+      model: selectedModel || "gpt-3.5-turbo",
       stream: false,
       messages: [
         {
           role: "system",
-          content: prompt,
+          content: systemPrompt,
         },
         {
           role: "user",
@@ -137,7 +139,7 @@ const getTopicFromAI = async (
     }),
   };
   const response = await fetch(
-    process.env.NEXT_PUBLIC_AI_URL + "/v1/chat/completions",
+    (upsreamUrl || process.env.NEXT_PUBLIC_AI_URL) + "/v1/chat/completions",
     requestOptions
   );
   const data = await response.json();
@@ -145,9 +147,6 @@ const getTopicFromAI = async (
   return topic; // 获取并返回回复
 };
 
-// 给getTopicFromAI函数创建别名
-// export const getFromAI = sendMessageToOpenAI;
-
 async function processResult(reader, decoder, editor) {
   let buffer = "";
   while (true) {
@@ -207,4 +206,4 @@ async function processResult(reader, decoder, editor) {
   }
 }
 
-export { getTopicFromAI, sendMessageToOpenAI };
+export { getAI, sendMessageToOpenAI };

sentry.client.config.ts

@@ -3,28 +3,29 @@
 // https://docs.sentry.io/platforms/javascript/guides/nextjs/
 
 import * as Sentry from "@sentry/nextjs";
+if (process.env.NODE_ENV === "production") {
+  Sentry.init({
+    dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
 
-Sentry.init({
+    // Adjust this value in production, or use tracesSampler for greater control
-  dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
+    tracesSampleRate: 1,
 
-  // Adjust this value in production, or use tracesSampler for greater control
+    // Setting this option to true will print useful information to the console while you're setting up Sentry.
-  tracesSampleRate: 1,
+    debug: false,
 
-  // Setting this option to true will print useful information to the console while you're setting up Sentry.
+    replaysOnErrorSampleRate: 1.0,
-  debug: false,
 
-  replaysOnErrorSampleRate: 1.0,
+    // This sets the sample rate to be 10%. You may want this to be 100% while
+    // in development and sample at a lower rate in production
+    replaysSessionSampleRate: 0.1,
 
-  // This sets the sample rate to be 10%. You may want this to be 100% while
+    // You can remove this option if you're not planning to use the Sentry Session Replay feature:
-  // in development and sample at a lower rate in production
+    integrations: [
-  replaysSessionSampleRate: 0.1,
+      Sentry.replayIntegration({
-
+        // Additional Replay configuration goes in here, for example:
-  // You can remove this option if you're not planning to use the Sentry Session Replay feature:
+        maskAllText: true,
-  integrations: [
+        blockAllMedia: true,
-    Sentry.replayIntegration({
+      }),
-      // Additional Replay configuration goes in here, for example:
+    ],
-      maskAllText: true,
+  });
-      blockAllMedia: true,
+}
-    }),
-  ],
-});

sentry.edge.config.ts

@@ -4,13 +4,14 @@
 // https://docs.sentry.io/platforms/javascript/guides/nextjs/
 
 import * as Sentry from "@sentry/nextjs";
+if (process.env.NODE_ENV === "production") {
+  Sentry.init({
+    dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
 
-Sentry.init({
+    // Adjust this value in production, or use tracesSampler for greater control
-  dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
+    tracesSampleRate: 1,
 
-  // Adjust this value in production, or use tracesSampler for greater control
+    // Setting this option to true will print useful information to the console while you're setting up Sentry.
-  tracesSampleRate: 1,
+    debug: false,
-
+  });
-  // Setting this option to true will print useful information to the console while you're setting up Sentry.
+}
-  debug: false,
-});

sentry.server.config.ts

@@ -3,13 +3,14 @@
 // https://docs.sentry.io/platforms/javascript/guides/nextjs/
 
 import * as Sentry from "@sentry/nextjs";
+if (process.env.NODE_ENV === "production") {
+  Sentry.init({
+    dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
 
-Sentry.init({
+    // Adjust this value in production, or use tracesSampler for greater control
-  dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
+    tracesSampleRate: 1,
 
-  // Adjust this value in production, or use tracesSampler for greater control
+    // Setting this option to true will print useful information to the console while you're setting up Sentry.
-  tracesSampleRate: 1,
+    debug: false,
-
+  });
-  // Setting this option to true will print useful information to the console while you're setting up Sentry.
+}
-  debug: false,
-});

utils/others/aiutils.ts

@@ -0,0 +1,51 @@
+// Path: utils/others/aiutils.ts
+import { getAI } from "@/components/chatAI";
+//判断返回的文献是否跟用户输入的主题相关
+export async function evaluateTopicMatch(
+  userMessage: any[],
+  apiKey: string,
+  upsreamUrl: string,
+  selectedModel: string,
+  topic: string
+): Promise<{ relevantPapers: string[]; nonRelevantPapers: string[] }> {
+  const prompt = "请判断文献是否跟用户输入的主题相关,只需要返回true或者false";
+  let relevantPapers: string[] = []; // 存储相关论文的数组
+  let nonRelevantPapers: string[] = []; // 存储不相关论文的数组
+
+  for (const paper of userMessage) {
+    const input = `user's topic:${topic}, \n paper's title: ${paper.title}, \n paper's abstract: ${paper.abstract}`;
+    const isRelevantResult = await getAI(
+      input,
+      prompt,
+      apiKey,
+      upsreamUrl,
+      selectedModel!
+    );
+    console.log("isRelevantResult", isRelevantResult);
+    // 尝试解析 JSON 结果，如果无法解析则直接使用结果字符串
+    let isRelevant;
+    try {
+      const parsedResult = JSON.parse(isRelevantResult);
+      isRelevant =
+        parsedResult === true || parsedResult.toLowerCase() === "true";
+    } catch {
+      isRelevant =
+        isRelevantResult.includes("true") || isRelevantResult.includes("True");
+    }
+
+    if (isRelevant) {
+      relevantPapers.push(paper); // 如果论文相关，则添加到数组中
+    } else {
+      nonRelevantPapers.push(paper); // 如果论文不相关，则添加到不相关论文数组中
+    }
+    console.log(
+      `这次有${nonRelevantPapers.length}篇文献没有通过相关性检查`,
+      nonRelevantPapers
+    );
+  }
+  //如果相关文献大于两片则缩减到两篇
+  if (relevantPapers.length > 2) {
+    relevantPapers = relevantPapers.slice(0, 2);
+  }
+  return { relevantPapers, nonRelevantPapers };
+}