Austin/paper-ai-release-24-07-21
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

feat: 可选的对文献相关性检验

Browse Source
This commit is contained in:
liuweiqing 2024-02-20 10:52:33 +08:00
parent b55cf4929a
commit 6cda6d176a
13 changed files with 204 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
app/[lng]/global-error.jsx → app/global-error.jsx
View File

3
app/i18n/locales/en/translation.json
View File

@ -27,5 +27,6 @@
"linuxdo": "linuxdo(Recommended)",
"custom": "Custom"
},
"鼠标点击段落中的上标跳转到文献引用？": "Click the superscript in the paragraph to jump to the reference?"
"鼠标点击段落中的上标跳转到文献引用？": "Click the superscript in the paragraph to jump to the reference?",
"是否检查文献与主题相关性(如果不相关则不会传给AI引用)": "Check the relevance of the literature to the topic (if it is not relevant, it will not be passed to the AI reference)"
}

3
app/i18n/locales/zh-CN/translation.json
View File

@ -28,5 +28,6 @@
"linuxdo": "linuxdo(第二个推荐)",
"custom": "自定义"
},
"鼠标点击段落中的上标跳转到文献引用？": "鼠标点击段落中的上标跳转到文献引用？"
"鼠标点击段落中的上标跳转到文献引用？": "鼠标点击段落中的上标跳转到文献引用？",
"是否检查文献与主题相关性(如果不相关则不会传给AI引用)": "是否检查文献与主题相关性（如果不相关则不会传给AI引用）"
}

1
app/store/index.ts
View File

@ -29,6 +29,7 @@ const statePersistConfig = {
"isVip",
"language",
"isJumpToReference",
"isEvaluateTopicMatch",
],
};

6
app/store/slices/stateSlice.ts
View File

@ -6,6 +6,7 @@ export interface APIState {
isVip: boolean;
language: string;
isJumpToReference: boolean;
isEvaluateTopicMatch: boolean;
}
const initialState: APIState = {
@ -15,6 +16,7 @@ const initialState: APIState = {
isVip: false,
language: "en",
isJumpToReference: false,
isEvaluateTopicMatch: false,
};
export const stateSlice = createSlice({
@ -45,6 +47,9 @@ export const stateSlice = createSlice({
setIsJumpToReference: (state, action: PayloadAction<boolean>) => {
state.isJumpToReference = action.payload;
},
setIsEvaluateTopicMatch: (state, action: PayloadAction<boolean>) => {
state.isEvaluateTopicMatch = action.payload;
},
},
});
@ -56,6 +61,7 @@ export const {
setIsVip,
setLanguage,
setIsJumpToReference,
setIsEvaluateTopicMatch,
} = stateSlice.actions;
export const stateReducer = stateSlice.reducer;

2
components/GetSemantic.tsx
View File

@ -32,7 +32,7 @@ async function getSemanticPapers(
params: {
query: query,
offset: offset,
limit: 2,
limit: limit,
year: year,
fields: "title,year,authors.name,abstract,venue,url,journal",
},

69
components/QuillEditor.tsx
View File

@ -10,7 +10,7 @@ import Link from "next/link";
import getArxivPapers from "./GetArxiv";
import getSemanticPapers from "./GetSemantic";
import { fetchPubMedData } from "./GetPubMed ";
import { getTopicFromAI, sendMessageToOpenAI } from "./chatAI";
import { getAI, sendMessageToOpenAI } from "./chatAI";
import {
getTextBeforeCursor,
convertToSuperscript,
@ -19,6 +19,7 @@ import {
getNumberBeforeCursor,
formatJournalReference,
} from "@/utils/others/quillutils";
import { evaluateTopicMatch } from "@/utils/others/aiutils";
//组件
import ExportDocx from "./Export";
import ReferenceList from "./ReferenceList";
@ -74,6 +75,9 @@ const QEditor = ({ lng }) => {
const isJumpToReference = useAppSelector(
(state) => state.state.isJumpToReference
);
const isEvaluateTopicMatch = useAppSelector(
(state) => state.state.isEvaluateTopicMatch
);
const [quill, setQuill] = useState<Quill | null>(null);
const contentUpdatedFromNetwork = useAppSelector(
(state) => state.state.contentUpdatedFromNetwork
@ -269,8 +273,14 @@ const QEditor = ({ lng }) => {
async function paper2AI(topic: string) {
quill!.setSelection(cursorPosition!, 0); // 将光标移动到原来的位置
let offset = -1;
if (generatedPaperNumber) offset = 0;
setOpenProgressBar(true);
if (generatedPaperNumber != 1) offset = 0; //如果生成的数量不为1，则从0开始
setOpenProgressBar(true); //开启进度条
//如果说要评估主题是否匹配的话,就要多获取一些文献
let limit = 2;
if (isEvaluateTopicMatch) {
limit = 4;
}
for (let i = 0; i < generatedPaperNumber!; i++) {
try {
if (!topic) {
@ -278,7 +288,13 @@ const QEditor = ({ lng }) => {
const prompt =
"As a topic extraction assistant, you can help me extract the current discussion of the paper topic, I will enter the content of the paper, you extract the paper topic , no more than two, Hyphenated query terms yield no matches (replace it with space to find matches) return format is: topic1 topic2";
const userMessage = getTextBeforeCursor(quill!, 2000);
topic = await getTopicFromAI(userMessage, prompt, apiKey);
topic = await getAI(
userMessage,
prompt,
apiKey,
upsreamUrl,
selectedModel!
);
console.log("topic in AI before removeSpecialCharacters", topic);
topic = removeSpecialCharacters(topic);
topic = topic.split(" ").slice(0, 2).join(" ");
@ -290,7 +306,19 @@ const QEditor = ({ lng }) => {
console.log("topic in AI", topic);
let rawData, dataString, newReferences;
if (selectedSource === "arxiv") {
rawData = await getArxivPapers(topic);
rawData = await getArxivPapers(topic, limit, offset);
//判断返回的文献是否跟用户输入的主题相关
if (isEvaluateTopicMatch) {
const { relevantPapers, nonRelevantPapers } =
await evaluateTopicMatch(
rawData,
apiKey,
upsreamUrl,
selectedModel!,
topic
);
rawData = relevantPapers;
}
console.log("arxiv rawdata:", rawData);
// 将 rawData 转换为引用数组
newReferences = rawData.map((entry: any) => ({
@ -305,7 +333,19 @@ const QEditor = ({ lng }) => {
})
.join("");
} else if (selectedSource === "semanticScholar") {
rawData = await getSemanticPapers(topic, "2015-2023", offset);
rawData = await getSemanticPapers(topic, "2015-2023", offset, limit);
//判断返回的文献是否跟用户输入的主题相关
if (isEvaluateTopicMatch) {
const { relevantPapers, nonRelevantPapers } =
await evaluateTopicMatch(
rawData,
apiKey,
upsreamUrl,
selectedModel!,
topic
);
rawData = relevantPapers;
}
// 将 rawData 转换为引用数组
newReferences = rawData.map((entry: any) => ({
url: entry.url,
@ -321,10 +361,22 @@ const QEditor = ({ lng }) => {
})
.join("");
} else if (selectedSource === "pubmed") {
rawData = await fetchPubMedData(topic, 2020, offset, 2);
rawData = await fetchPubMedData(topic, 2020, offset, limit);
if (!rawData) {
throw new Error("未搜索到文献 from PubMed.");
}
//判断返回的文献是否跟用户输入的主题相关
if (isEvaluateTopicMatch) {
const { relevantPapers, nonRelevantPapers } =
await evaluateTopicMatch(
rawData,
apiKey,
upsreamUrl,
selectedModel!,
topic
);
rawData = relevantPapers;
}
newReferences = rawData.map((entry: any) => ({
id: entry.id, // 文章的 PubMed ID
title: entry.title, // 文章的标题
@ -336,9 +388,8 @@ const QEditor = ({ lng }) => {
source: "PubMed", // 指示这些引用来自 PubMed
}));
// 打印或进一步处理 newReferences
// 打印 newReferences
console.log(newReferences);
dataString = rawData
.map((entry: any) => {
return `Time: ${entry.year}\nTitle: ${entry.title}\nSummary: ${entry.abstract}\n\n`;

38
components/Settings.tsx
View File

@ -7,7 +7,10 @@ import {
setUpsreamUrl,
setSystemPrompt,
} from "@/app/store/slices/authSlice";
import { setIsJumpToReference } from "@/app/store/slices/stateSlice";
import {
setIsJumpToReference,
setIsEvaluateTopicMatch,
} from "@/app/store/slices/stateSlice";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faArrowLeft } from "@fortawesome/free-solid-svg-icons";
import Link from "next/link";
@ -53,14 +56,16 @@ const Settings = ({ lng }: { lng: string }) => {
const isJumpToReference = useAppSelector(
(state) => state.state.isJumpToReference
);
const isEvaluateTopicMatch = useAppSelector(
(state) => state.state.isEvaluateTopicMatch
);
//state
const [userConfigNumber, setUserConfigNumber] = useLocalStorage(
"userConfigNumber",
"2"
);
const toggleSwitch = () => {
dispatch(setIsJumpToReference(!isJumpToReference));
const toggleSwitch = (currentState: any, setState: any) => {
setState(!currentState);
};
return (
<div className="max-w-md rounded overflow-hidden shadow-lg bg-blue-gray-100 z-1000 mx-auto ">
@ -153,7 +158,11 @@ const Settings = ({ lng }: { lng: string }) => {
type="checkbox"
className="sr-only peer"
checked={isJumpToReference}
onChange={toggleSwitch}
onChange={() =>
toggleSwitch(isJumpToReference, (value: any) =>
dispatch(setIsJumpToReference(value))
)
}
/>
<div className="w-10 h-4 bg-gray-200 rounded-full peer-checked:bg-blue-600 peer-focus:ring-4 peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800 transition-colors ease-in-out duration-200"></div>
<span
@ -163,6 +172,25 @@ const Settings = ({ lng }: { lng: string }) => {
></span>
{t("鼠标点击段落中的上标跳转到文献引用？")}
</label>
<label className="relative inline-flex items-center cursor-pointer">
<input
type="checkbox"
className="sr-only peer"
checked={isEvaluateTopicMatch}
onChange={() =>
toggleSwitch(isEvaluateTopicMatch, (value: any) =>
dispatch(setIsEvaluateTopicMatch(value))
)
}
/>
<div className="w-10 h-4 bg-gray-200 rounded-full peer-checked:bg-blue-600 peer-focus:ring-4 peer-focus:ring-blue-300 dark:peer-focus:ring-blue-800 transition-colors ease-in-out duration-200"></div>
<span
className={`absolute block bg-white w-3 h-3 rounded-full transition ease-in-out duration-200 transform ${
isJumpToReference ? "translate-x-6" : "translate-x-1"
} -translate-y-1/2 top-1/2`}
></span>
{t("是否检查文献与主题相关性(如果不相关则不会传给AI引用)")}
</label>
</div>
);
};

19
components/chatAI.tsx
View File

@ -105,10 +105,12 @@ const sendMessageToOpenAI = async (
}
};
const getTopicFromAI = async (
const getAI = async (
userMessage: string,
prompt: string,
apiKey: string
systemPrompt: string,
apiKey: string,
upsreamUrl: string,
selectedModel: string
) => {
// 设置API请求参数
const requestOptions = {
@ -122,12 +124,12 @@ const getTopicFromAI = async (
: process.env.NEXT_PUBLIC_OPENAI_API_KEY),
},
body: JSON.stringify({
model: "gpt-3.5-turbo",
model: selectedModel || "gpt-3.5-turbo",
stream: false,
messages: [
{
role: "system",
content: prompt,
content: systemPrompt,
},
{
role: "user",
@ -137,7 +139,7 @@ const getTopicFromAI = async (
}),
};
const response = await fetch(
process.env.NEXT_PUBLIC_AI_URL + "/v1/chat/completions",
(upsreamUrl || process.env.NEXT_PUBLIC_AI_URL) + "/v1/chat/completions",
requestOptions
);
const data = await response.json();
@ -145,9 +147,6 @@ const getTopicFromAI = async (
return topic; // 获取并返回回复
};
// 给getTopicFromAI函数创建别名
// export const getFromAI = sendMessageToOpenAI;
async function processResult(reader, decoder, editor) {
let buffer = "";
while (true) {
@ -207,4 +206,4 @@ async function processResult(reader, decoder, editor) {
}
}
export { getTopicFromAI, sendMessageToOpenAI };
export { getAI, sendMessageToOpenAI };

41
sentry.client.config.ts
View File

@ -3,28 +3,29 @@
// https://docs.sentry.io/platforms/javascript/guides/nextjs/
import * as Sentry from "@sentry/nextjs";
if (process.env.NODE_ENV === "production") {
Sentry.init({
dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
Sentry.init({
dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
// Adjust this value in production, or use tracesSampler for greater control
tracesSampleRate: 1,
// Adjust this value in production, or use tracesSampler for greater control
tracesSampleRate: 1,
// Setting this option to true will print useful information to the console while you're setting up Sentry.
debug: false,
// Setting this option to true will print useful information to the console while you're setting up Sentry.
debug: false,
replaysOnErrorSampleRate: 1.0,
replaysOnErrorSampleRate: 1.0,
// This sets the sample rate to be 10%. You may want this to be 100% while
// in development and sample at a lower rate in production
replaysSessionSampleRate: 0.1,
// This sets the sample rate to be 10%. You may want this to be 100% while
// in development and sample at a lower rate in production
replaysSessionSampleRate: 0.1,
// You can remove this option if you're not planning to use the Sentry Session Replay feature:
integrations: [
Sentry.replayIntegration({
// Additional Replay configuration goes in here, for example:
maskAllText: true,
blockAllMedia: true,
}),
],
});
// You can remove this option if you're not planning to use the Sentry Session Replay feature:
integrations: [
Sentry.replayIntegration({
// Additional Replay configuration goes in here, for example:
maskAllText: true,
blockAllMedia: true,
}),
],
});
}

17
sentry.edge.config.ts
View File

@ -4,13 +4,14 @@
// https://docs.sentry.io/platforms/javascript/guides/nextjs/
import * as Sentry from "@sentry/nextjs";
if (process.env.NODE_ENV === "production") {
Sentry.init({
dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
Sentry.init({
dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
// Adjust this value in production, or use tracesSampler for greater control
tracesSampleRate: 1,
// Adjust this value in production, or use tracesSampler for greater control
tracesSampleRate: 1,
// Setting this option to true will print useful information to the console while you're setting up Sentry.
debug: false,
});
// Setting this option to true will print useful information to the console while you're setting up Sentry.
debug: false,
});
}

17
sentry.server.config.ts
View File

@ -3,13 +3,14 @@
// https://docs.sentry.io/platforms/javascript/guides/nextjs/
import * as Sentry from "@sentry/nextjs";
if (process.env.NODE_ENV === "production") {
Sentry.init({
dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
Sentry.init({
dsn: "https://523c4056ba48d012c62a377dfc49f647@o4506728662564864.ingest.sentry.io/4506728672264192",
// Adjust this value in production, or use tracesSampler for greater control
tracesSampleRate: 1,
// Adjust this value in production, or use tracesSampler for greater control
tracesSampleRate: 1,
// Setting this option to true will print useful information to the console while you're setting up Sentry.
debug: false,
});
// Setting this option to true will print useful information to the console while you're setting up Sentry.
debug: false,
});
}

51
utils/others/aiutils.ts Normal file
View File

@ -0,0 +1,51 @@
// Path: utils/others/aiutils.ts
import { getAI } from "@/components/chatAI";
//判断返回的文献是否跟用户输入的主题相关
export async function evaluateTopicMatch(
userMessage: any[],
apiKey: string,
upsreamUrl: string,
selectedModel: string,
topic: string
): Promise<{ relevantPapers: string[]; nonRelevantPapers: string[] }> {
const prompt = "请判断文献是否跟用户输入的主题相关,只需要返回true或者false";
let relevantPapers: string[] = []; // 存储相关论文的数组
let nonRelevantPapers: string[] = []; // 存储不相关论文的数组
for (const paper of userMessage) {
const input = `user's topic:${topic}, \n paper's title: ${paper.title}, \n paper's abstract: ${paper.abstract}`;
const isRelevantResult = await getAI(
input,
prompt,
apiKey,
upsreamUrl,
selectedModel!
);
console.log("isRelevantResult", isRelevantResult);
// 尝试解析 JSON 结果，如果无法解析则直接使用结果字符串
let isRelevant;
try {
const parsedResult = JSON.parse(isRelevantResult);
isRelevant =
parsedResult === true || parsedResult.toLowerCase() === "true";
} catch {
isRelevant =
isRelevantResult.includes("true") || isRelevantResult.includes("True");
}
if (isRelevant) {
relevantPapers.push(paper); // 如果论文相关，则添加到数组中
} else {
nonRelevantPapers.push(paper); // 如果论文不相关，则添加到不相关论文数组中
}
console.log(
`这次有${nonRelevantPapers.length}篇文献没有通过相关性检查`,
nonRelevantPapers
);
}
//如果相关文献大于两片则缩减到两篇
if (relevantPapers.length > 2) {
relevantPapers = relevantPapers.slice(0, 2);
}
return { relevantPapers, nonRelevantPapers };
}