From f257f2c39647ed1e021880f298cc5c17f9d2c64a Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:02:29 +0800 Subject: [PATCH] Knowledge optimization (#3755) Co-authored-by: crazywoola <427733928@qq.com> Co-authored-by: JzoNg --- api/controllers/console/__init__.py | 3 + api/controllers/console/app/app.py | 23 +- api/controllers/console/datasets/datasets.py | 26 +- .../console/datasets/datasets_document.py | 47 ++- api/controllers/console/tag/tags.py | 159 ++++++++++ .../service_api/dataset/dataset.py | 5 +- .../datasource/vdb/milvus/milvus_vector.py | 36 ++- .../datasource/vdb/qdrant/qdrant_vector.py | 82 ++++-- .../vdb/weaviate/weaviate_vector.py | 37 +-- api/fields/app_fields.py | 10 +- api/fields/dataset_fields.py | 8 +- api/fields/tag_fields.py | 8 + ...3c7cac9521c6_add_tags_and_binding_table.py | 62 ++++ api/models/dataset.py | 16 +- api/models/model.py | 65 ++++- api/services/app_service.py | 11 +- api/services/dataset_service.py | 105 ++++++- api/services/tag_service.py | 161 +++++++++++ api/tasks/clean_dataset_task.py | 4 + api/tasks/deal_dataset_vector_index_task.py | 33 +++ api/tasks/duplicate_document_indexing_task.py | 94 ++++++ api/tasks/retry_document_indexing_task.py | 91 ++++++ web/app/(commonLayout)/apps/AppCard.tsx | 99 +++++-- web/app/(commonLayout)/apps/Apps.tsx | 58 ++-- web/app/(commonLayout)/datasets/Container.tsx | 38 ++- .../(commonLayout)/datasets/DatasetCard.tsx | 210 ++++++++++---- web/app/(commonLayout)/datasets/Datasets.tsx | 35 ++- .../datasets/NewDatasetCard.tsx | 21 +- .../datasets/assets/application.svg | 6 - .../(commonLayout)/datasets/assets/doc.svg | 3 - .../(commonLayout)/datasets/assets/text.svg | 3 - web/app/(commonLayout)/list.module.css | 12 - .../line/financeAndECommerce/tag-01.svg | 10 + .../line/financeAndECommerce/tag-03.svg | 5 + .../line/financeAndECommerce/Tag01.json | 66 +++++ .../vender/line/financeAndECommerce/Tag01.tsx | 16 ++ .../line/financeAndECommerce/Tag03.json | 39 +++ .../vender/line/financeAndECommerce/Tag03.tsx | 16 ++ .../vender/line/financeAndECommerce/index.ts | 2 + web/app/components/base/popover/index.tsx | 10 +- .../components/base/retry-button/index.tsx | 85 ++++++ .../base/retry-button/style.module.css | 4 + .../components/base/search-input/index.tsx | 66 +++++ .../base/tag-management/constant.ts | 6 + .../components/base/tag-management/filter.tsx | 142 +++++++++ .../components/base/tag-management/index.tsx | 93 ++++++ .../base/tag-management/selector.tsx | 272 ++++++++++++++++++ .../components/base/tag-management/store.ts | 19 ++ .../base/tag-management/style.module.css | 3 + .../base/tag-management/tag-item-editor.tsx | 147 ++++++++++ .../base/tag-management/tag-remove-modal.tsx | 50 ++++ .../components/datasets/documents/index.tsx | 22 +- .../components/datasets/documents/list.tsx | 2 +- .../datasets/rename-modal/index.tsx | 106 +++++++ .../datasets/settings/form/index.tsx | 29 +- web/i18n/de-DE/common.ts | 17 ++ web/i18n/de-DE/dataset.ts | 4 +- web/i18n/en-US/common.ts | 17 ++ web/i18n/en-US/dataset.ts | 6 +- web/i18n/fr-FR/common.ts | 17 ++ web/i18n/fr-FR/dataset.ts | 4 +- web/i18n/ja-JP/common.ts | 17 ++ web/i18n/ja-JP/dataset.ts | 4 +- web/i18n/pt-BR/common.ts | 17 ++ web/i18n/pt-BR/dataset.ts | 4 +- web/i18n/uk-UA/common.ts | 17 ++ web/i18n/uk-UA/dataset.ts | 2 + web/i18n/vi-VN/common.ts | 17 ++ web/i18n/vi-VN/dataset.ts | 4 +- web/i18n/zh-Hans/common.ts | 17 ++ web/i18n/zh-Hans/dataset.ts | 4 +- web/models/datasets.ts | 7 + web/service/datasets.ts | 16 +- web/service/tag.ts | 47 +++ web/types/app.ts | 3 + 75 files changed, 2756 insertions(+), 266 deletions(-) create mode 100644 api/controllers/console/tag/tags.py create mode 100644 api/fields/tag_fields.py create mode 100644 api/migrations/versions/3c7cac9521c6_add_tags_and_binding_table.py create mode 100644 api/services/tag_service.py create mode 100644 api/tasks/duplicate_document_indexing_task.py create mode 100644 api/tasks/retry_document_indexing_task.py delete mode 100644 web/app/(commonLayout)/datasets/assets/application.svg delete mode 100644 web/app/(commonLayout)/datasets/assets/doc.svg delete mode 100644 web/app/(commonLayout)/datasets/assets/text.svg create mode 100644 web/app/components/base/icons/assets/vender/line/financeAndECommerce/tag-01.svg create mode 100644 web/app/components/base/icons/assets/vender/line/financeAndECommerce/tag-03.svg create mode 100644 web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag01.json create mode 100644 web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag01.tsx create mode 100644 web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag03.json create mode 100644 web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag03.tsx create mode 100644 web/app/components/base/retry-button/index.tsx create mode 100644 web/app/components/base/retry-button/style.module.css create mode 100644 web/app/components/base/search-input/index.tsx create mode 100644 web/app/components/base/tag-management/constant.ts create mode 100644 web/app/components/base/tag-management/filter.tsx create mode 100644 web/app/components/base/tag-management/index.tsx create mode 100644 web/app/components/base/tag-management/selector.tsx create mode 100644 web/app/components/base/tag-management/store.ts create mode 100644 web/app/components/base/tag-management/style.module.css create mode 100644 web/app/components/base/tag-management/tag-item-editor.tsx create mode 100644 web/app/components/base/tag-management/tag-remove-modal.tsx create mode 100644 web/app/components/datasets/rename-modal/index.tsx create mode 100644 web/service/tag.ts diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index 39c96d9673..498557cd51 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -53,5 +53,8 @@ from .explore import ( workflow, ) +# Import tag controllers +from .tag import tags + # Import workspace controllers from .workspace import account, members, model_providers, models, tool_providers, workspace diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index fb9c2c23ca..21660e00de 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -1,18 +1,25 @@ +import json +import uuid + from flask_login import current_user -from flask_restful import Resource, inputs, marshal_with, reqparse -from werkzeug.exceptions import BadRequest, Forbidden +from flask_restful import Resource, inputs, marshal, marshal_with, reqparse +from werkzeug.exceptions import BadRequest, Forbidden, abort from controllers.console import api from controllers.console.app.wraps import get_app_model from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check +from core.tools.tool_manager import ToolManager +from core.tools.utils.configuration import ToolParameterConfigurationManager from fields.app_fields import ( app_detail_fields, app_detail_fields_with_site, app_pagination_fields, ) from libs.login import login_required +from models.model import App, AppMode, AppModelConfig from services.app_service import AppService +from services.tag_service import TagService ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion'] @@ -22,21 +29,29 @@ class AppListApi(Resource): @setup_required @login_required @account_initialization_required - @marshal_with(app_pagination_fields) def get(self): """Get app list""" + def uuid_list(value): + try: + return [str(uuid.UUID(v)) for v in value.split(',')] + except ValueError: + abort(400, message="Invalid UUID format in tag_ids.") parser = reqparse.RequestParser() parser.add_argument('page', type=inputs.int_range(1, 99999), required=False, default=1, location='args') parser.add_argument('limit', type=inputs.int_range(1, 100), required=False, default=20, location='args') parser.add_argument('mode', type=str, choices=['chat', 'workflow', 'agent-chat', 'channel', 'all'], default='all', location='args', required=False) parser.add_argument('name', type=str, location='args', required=False) + parser.add_argument('tag_ids', type=uuid_list, location='args', required=False) + args = parser.parse_args() # get app list app_service = AppService() app_pagination = app_service.get_paginate_apps(current_user.current_tenant_id, args) + if not app_pagination: + return {'data': [], 'total': 0, 'page': 1, 'limit': 20, 'has_more': False} - return app_pagination + return marshal(app_pagination, app_pagination_fields) @setup_required @login_required diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index e633631c42..c0027f3c47 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -48,11 +48,14 @@ class DatasetListApi(Resource): limit = request.args.get('limit', default=20, type=int) ids = request.args.getlist('ids') provider = request.args.get('provider', default="vendor") + search = request.args.get('keyword', default=None, type=str) + tag_ids = request.args.getlist('tag_ids') + if ids: datasets, total = DatasetService.get_datasets_by_ids(ids, current_user.current_tenant_id) else: datasets, total = DatasetService.get_datasets(page, limit, provider, - current_user.current_tenant_id, current_user) + current_user.current_tenant_id, current_user, search, tag_ids) # check embedding setting provider_manager = ProviderManager() @@ -184,6 +187,10 @@ class DatasetApi(Resource): help='Invalid indexing technique.') parser.add_argument('permission', type=str, location='json', choices=( 'only_me', 'all_team_members'), help='Invalid permission.') + parser.add_argument('embedding_model', type=str, + location='json', help='Invalid embedding model.') + parser.add_argument('embedding_model_provider', type=str, + location='json', help='Invalid embedding model provider.') parser.add_argument('retrieval_model', type=dict, location='json', help='Invalid retrieval model.') args = parser.parse_args() @@ -506,10 +513,27 @@ class DatasetRetrievalSettingMockApi(Resource): else: raise ValueError("Unsupported vector db type.") +class DatasetErrorDocs(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self, dataset_id): + dataset_id_str = str(dataset_id) + dataset = DatasetService.get_dataset(dataset_id_str) + if dataset is None: + raise NotFound("Dataset not found.") + results = DocumentService.get_error_documents_by_dataset_id(dataset_id_str) + + return { + 'data': [marshal(item, document_status_fields) for item in results], + 'total': len(results) + }, 200 + api.add_resource(DatasetListApi, '/datasets') api.add_resource(DatasetApi, '/datasets/') api.add_resource(DatasetQueryApi, '/datasets//queries') +api.add_resource(DatasetErrorDocs, '/datasets//error-docs') api.add_resource(DatasetIndexingEstimateApi, '/datasets/indexing-estimate') api.add_resource(DatasetRelatedAppListApi, '/datasets//related-apps') api.add_resource(DatasetIndexingStatusApi, '/datasets//indexing-status') diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 3d6daa7682..d8bd6ad78f 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime, timezone from flask import request @@ -233,7 +234,7 @@ class DatasetDocumentListApi(Resource): location='json') parser.add_argument('data_source', type=dict, required=False, location='json') parser.add_argument('process_rule', type=dict, required=False, location='json') - parser.add_argument('duplicate', type=bool, nullable=False, location='json') + parser.add_argument('duplicate', type=bool, default=True, nullable=False, location='json') parser.add_argument('original_document_id', type=str, required=False, location='json') parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json') parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False, @@ -883,6 +884,49 @@ class DocumentRecoverApi(DocumentResource): return {'result': 'success'}, 204 +class DocumentRetryApi(DocumentResource): + @setup_required + @login_required + @account_initialization_required + def post(self, dataset_id): + """retry document.""" + + parser = reqparse.RequestParser() + parser.add_argument('document_ids', type=list, required=True, nullable=False, + location='json') + args = parser.parse_args() + dataset_id = str(dataset_id) + dataset = DatasetService.get_dataset(dataset_id) + retry_documents = [] + if not dataset: + raise NotFound('Dataset not found.') + for document_id in args['document_ids']: + try: + document_id = str(document_id) + + document = DocumentService.get_document(dataset.id, document_id) + + # 404 if document not found + if document is None: + raise NotFound("Document Not Exists.") + + # 403 if document is archived + if DocumentService.check_archived(document): + raise ArchivedDocumentImmutableError() + + # 400 if document is completed + if document.indexing_status == 'completed': + raise DocumentAlreadyFinishedError() + retry_documents.append(document) + except Exception as e: + logging.error(f"Document {document_id} retry failed: {str(e)}") + continue + # retry document + DocumentService.retry_document(dataset_id, retry_documents) + + return {'result': 'success'}, 204 + + api.add_resource(GetProcessRuleApi, '/datasets/process-rule') api.add_resource(DatasetDocumentListApi, '/datasets//documents') @@ -908,3 +952,4 @@ api.add_resource(DocumentStatusApi, '/datasets//documents//status/') api.add_resource(DocumentPauseApi, '/datasets//documents//processing/pause') api.add_resource(DocumentRecoverApi, '/datasets//documents//processing/resume') +api.add_resource(DocumentRetryApi, '/datasets//retry') diff --git a/api/controllers/console/tag/tags.py b/api/controllers/console/tag/tags.py new file mode 100644 index 0000000000..5dcba301cb --- /dev/null +++ b/api/controllers/console/tag/tags.py @@ -0,0 +1,159 @@ +from flask import request +from flask_login import current_user +from flask_restful import Resource, marshal_with, reqparse +from werkzeug.exceptions import Forbidden + +from controllers.console import api +from controllers.console.setup import setup_required +from controllers.console.wraps import account_initialization_required +from fields.tag_fields import tag_fields +from libs.login import login_required +from models.model import Tag +from services.tag_service import TagService + + +def _validate_name(name): + if not name or len(name) < 1 or len(name) > 40: + raise ValueError('Name must be between 1 to 50 characters.') + return name + + +class TagListApi(Resource): + + @setup_required + @login_required + @account_initialization_required + @marshal_with(tag_fields) + def get(self): + tag_type = request.args.get('type', type=str) + keyword = request.args.get('keyword', default=None, type=str) + tags = TagService.get_tags(tag_type, current_user.current_tenant_id, keyword) + + return tags, 200 + + @setup_required + @login_required + @account_initialization_required + def post(self): + # The role of the current user in the ta table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() + + parser = reqparse.RequestParser() + parser.add_argument('name', nullable=False, required=True, + help='Name must be between 1 to 50 characters.', + type=_validate_name) + parser.add_argument('type', type=str, location='json', + choices=Tag.TAG_TYPE_LIST, + nullable=True, + help='Invalid tag type.') + args = parser.parse_args() + tag = TagService.save_tags(args) + + response = { + 'id': tag.id, + 'name': tag.name, + 'type': tag.type, + 'binding_count': 0 + } + + return response, 200 + + +class TagUpdateDeleteApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def patch(self, tag_id): + tag_id = str(tag_id) + # The role of the current user in the ta table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() + + parser = reqparse.RequestParser() + parser.add_argument('name', nullable=False, required=True, + help='Name must be between 1 to 50 characters.', + type=_validate_name) + args = parser.parse_args() + tag = TagService.update_tags(args, tag_id) + + binding_count = TagService.get_tag_binding_count(tag_id) + + response = { + 'id': tag.id, + 'name': tag.name, + 'type': tag.type, + 'binding_count': binding_count + } + + return response, 200 + + @setup_required + @login_required + @account_initialization_required + def delete(self, tag_id): + tag_id = str(tag_id) + # The role of the current user in the ta table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() + + TagService.delete_tag(tag_id) + + return 200 + + +class TagBindingCreateApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def post(self): + # The role of the current user in the ta table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() + + parser = reqparse.RequestParser() + parser.add_argument('tag_ids', type=list, nullable=False, required=True, location='json', + help='Tag IDs is required.') + parser.add_argument('target_id', type=str, nullable=False, required=True, location='json', + help='Target ID is required.') + parser.add_argument('type', type=str, location='json', + choices=Tag.TAG_TYPE_LIST, + nullable=True, + help='Invalid tag type.') + args = parser.parse_args() + TagService.save_tag_binding(args) + + return 200 + + +class TagBindingDeleteApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def post(self): + # The role of the current user in the ta table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() + + parser = reqparse.RequestParser() + parser.add_argument('tag_id', type=str, nullable=False, required=True, + help='Tag ID is required.') + parser.add_argument('target_id', type=str, nullable=False, required=True, + help='Target ID is required.') + parser.add_argument('type', type=str, location='json', + choices=Tag.TAG_TYPE_LIST, + nullable=True, + help='Invalid tag type.') + args = parser.parse_args() + TagService.delete_tag_binding(args) + + return 200 + + +api.add_resource(TagListApi, '/tags') +api.add_resource(TagUpdateDeleteApi, '/tags/') +api.add_resource(TagBindingCreateApi, '/tag-bindings/create') +api.add_resource(TagBindingDeleteApi, '/tag-bindings/remove') diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index 60c7ca4549..bf08291d7b 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -26,8 +26,11 @@ class DatasetApi(DatasetApiResource): page = request.args.get('page', default=1, type=int) limit = request.args.get('limit', default=20, type=int) provider = request.args.get('provider', default="vendor") + search = request.args.get('keyword', default=None, type=str) + tag_ids = request.args.getlist('tag_ids') + datasets, total = DatasetService.get_datasets(page, limit, provider, - tenant_id, current_user) + tenant_id, current_user, search, tag_ids) # check embedding setting provider_manager = ProviderManager() configurations = provider_manager.get_configurations( diff --git a/api/core/rag/datasource/vdb/milvus/milvus_vector.py b/api/core/rag/datasource/vdb/milvus/milvus_vector.py index 42ba5224a4..aa3625a5a5 100644 --- a/api/core/rag/datasource/vdb/milvus/milvus_vector.py +++ b/api/core/rag/datasource/vdb/milvus/milvus_vector.py @@ -110,19 +110,37 @@ class MilvusVector(BaseVector): return None def delete_by_metadata_field(self, key: str, value: str): + alias = uuid4().hex + if self._client_config.secure: + uri = "https://" + str(self._client_config.host) + ":" + str(self._client_config.port) + else: + uri = "http://" + str(self._client_config.host) + ":" + str(self._client_config.port) + connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password) - ids = self.get_ids_by_metadata_field(key, value) - if ids: - self._client.delete(collection_name=self._collection_name, pks=ids) + from pymilvus import utility + if utility.has_collection(self._collection_name, using=alias): + + ids = self.get_ids_by_metadata_field(key, value) + if ids: + self._client.delete(collection_name=self._collection_name, pks=ids) def delete_by_ids(self, doc_ids: list[str]) -> None: + alias = uuid4().hex + if self._client_config.secure: + uri = "https://" + str(self._client_config.host) + ":" + str(self._client_config.port) + else: + uri = "http://" + str(self._client_config.host) + ":" + str(self._client_config.port) + connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password) - result = self._client.query(collection_name=self._collection_name, - filter=f'metadata["doc_id"] in {doc_ids}', - output_fields=["id"]) - if result: - ids = [item["id"] for item in result] - self._client.delete(collection_name=self._collection_name, pks=ids) + from pymilvus import utility + if utility.has_collection(self._collection_name, using=alias): + + result = self._client.query(collection_name=self._collection_name, + filter=f'metadata["doc_id"] in {doc_ids}', + output_fields=["id"]) + if result: + ids = [item["id"] for item in result] + self._client.delete(collection_name=self._collection_name, pks=ids) def delete(self) -> None: alias = uuid4().hex diff --git a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py index 41e8c6154a..2d0bb90094 100644 --- a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py +++ b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py @@ -217,29 +217,38 @@ class QdrantVector(BaseVector): def delete_by_metadata_field(self, key: str, value: str): from qdrant_client.http import models + from qdrant_client.http.exceptions import UnexpectedResponse - filter = models.Filter( - must=[ - models.FieldCondition( - key=f"metadata.{key}", - match=models.MatchValue(value=value), + try: + filter = models.Filter( + must=[ + models.FieldCondition( + key=f"metadata.{key}", + match=models.MatchValue(value=value), + ), + ], + ) + + self._reload_if_needed() + + self._client.delete( + collection_name=self._collection_name, + points_selector=FilterSelector( + filter=filter ), - ], - ) - - self._reload_if_needed() - - self._client.delete( - collection_name=self._collection_name, - points_selector=FilterSelector( - filter=filter - ), - ) + ) + except UnexpectedResponse as e: + # Collection does not exist, so return + if e.status_code == 404: + return + # Some other error occurred, so re-raise the exception + else: + raise e def delete(self): from qdrant_client.http import models from qdrant_client.http.exceptions import UnexpectedResponse - + try: filter = models.Filter( must=[ @@ -257,29 +266,40 @@ class QdrantVector(BaseVector): ) except UnexpectedResponse as e: # Collection does not exist, so return - if e.status_code == 404: + if e.status_code == 404: return # Some other error occurred, so re-raise the exception else: raise e + def delete_by_ids(self, ids: list[str]) -> None: from qdrant_client.http import models + from qdrant_client.http.exceptions import UnexpectedResponse + for node_id in ids: - filter = models.Filter( - must=[ - models.FieldCondition( - key="metadata.doc_id", - match=models.MatchValue(value=node_id), + try: + filter = models.Filter( + must=[ + models.FieldCondition( + key="metadata.doc_id", + match=models.MatchValue(value=node_id), + ), + ], + ) + self._client.delete( + collection_name=self._collection_name, + points_selector=FilterSelector( + filter=filter ), - ], - ) - self._client.delete( - collection_name=self._collection_name, - points_selector=FilterSelector( - filter=filter - ), - ) + ) + except UnexpectedResponse as e: + # Collection does not exist, so return + if e.status_code == 404: + return + # Some other error occurred, so re-raise the exception + else: + raise e def text_exists(self, id: str) -> bool: all_collection_name = [] diff --git a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py index 59fbaeee6a..93c7480f8b 100644 --- a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py +++ b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py @@ -121,18 +121,20 @@ class WeaviateVector(BaseVector): return ids def delete_by_metadata_field(self, key: str, value: str): + # check whether the index already exists + schema = self._default_schema(self._collection_name) + if self._client.schema.contains(schema): + where_filter = { + "operator": "Equal", + "path": [key], + "valueText": value + } - where_filter = { - "operator": "Equal", - "path": [key], - "valueText": value - } - - self._client.batch.delete_objects( - class_name=self._collection_name, - where=where_filter, - output='minimal' - ) + self._client.batch.delete_objects( + class_name=self._collection_name, + where=where_filter, + output='minimal' + ) def delete(self): # check whether the index already exists @@ -163,11 +165,14 @@ class WeaviateVector(BaseVector): return True def delete_by_ids(self, ids: list[str]) -> None: - for uuid in ids: - self._client.data_object.delete( - class_name=self._collection_name, - uuid=uuid, - ) + # check whether the index already exists + schema = self._default_schema(self._collection_name) + if self._client.schema.contains(schema): + for uuid in ids: + self._client.data_object.delete( + class_name=self._collection_name, + uuid=uuid, + ) def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: """Look up similar documents by embedding vector in Weaviate.""" diff --git a/api/fields/app_fields.py b/api/fields/app_fields.py index 3c56e680de..c7cfdd7939 100644 --- a/api/fields/app_fields.py +++ b/api/fields/app_fields.py @@ -62,6 +62,12 @@ model_config_partial_fields = { 'pre_prompt': fields.String, } +tag_fields = { + 'id': fields.String, + 'name': fields.String, + 'type': fields.String +} + app_partial_fields = { 'id': fields.String, 'name': fields.String, @@ -70,9 +76,11 @@ app_partial_fields = { 'icon': fields.String, 'icon_background': fields.String, 'model_config': fields.Nested(model_config_partial_fields, attribute='app_model_config', allow_null=True), - 'created_at': TimestampField + 'created_at': TimestampField, + 'tags': fields.List(fields.Nested(tag_fields)) } + app_pagination_fields = { 'page': fields.Integer, 'limit': fields.Integer(attribute='per_page'), diff --git a/api/fields/dataset_fields.py b/api/fields/dataset_fields.py index eb2ccb8f9f..50c5f43540 100644 --- a/api/fields/dataset_fields.py +++ b/api/fields/dataset_fields.py @@ -27,6 +27,11 @@ dataset_retrieval_model_fields = { 'score_threshold': fields.Float } +tag_fields = { + 'id': fields.String, + 'name': fields.String, + 'type': fields.String +} dataset_detail_fields = { 'id': fields.String, @@ -46,7 +51,8 @@ dataset_detail_fields = { 'embedding_model': fields.String, 'embedding_model_provider': fields.String, 'embedding_available': fields.Boolean, - 'retrieval_model_dict': fields.Nested(dataset_retrieval_model_fields) + 'retrieval_model_dict': fields.Nested(dataset_retrieval_model_fields), + 'tags': fields.List(fields.Nested(tag_fields)) } dataset_query_detail_fields = { diff --git a/api/fields/tag_fields.py b/api/fields/tag_fields.py new file mode 100644 index 0000000000..f7e030b738 --- /dev/null +++ b/api/fields/tag_fields.py @@ -0,0 +1,8 @@ +from flask_restful import fields + +tag_fields = { + 'id': fields.String, + 'name': fields.String, + 'type': fields.String, + 'binding_count': fields.String +} \ No newline at end of file diff --git a/api/migrations/versions/3c7cac9521c6_add_tags_and_binding_table.py b/api/migrations/versions/3c7cac9521c6_add_tags_and_binding_table.py new file mode 100644 index 0000000000..5f11880683 --- /dev/null +++ b/api/migrations/versions/3c7cac9521c6_add_tags_and_binding_table.py @@ -0,0 +1,62 @@ +"""add-tags-and-binding-table + +Revision ID: 3c7cac9521c6 +Revises: c3311b089690 +Create Date: 2024-04-11 06:17:34.278594 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '3c7cac9521c6' +down_revision = 'c3311b089690' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('tag_bindings', + sa.Column('id', postgresql.UUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('tenant_id', postgresql.UUID(), nullable=True), + sa.Column('tag_id', postgresql.UUID(), nullable=True), + sa.Column('target_id', postgresql.UUID(), nullable=True), + sa.Column('created_by', postgresql.UUID(), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.PrimaryKeyConstraint('id', name='tag_binding_pkey') + ) + with op.batch_alter_table('tag_bindings', schema=None) as batch_op: + batch_op.create_index('tag_bind_tag_id_idx', ['tag_id'], unique=False) + batch_op.create_index('tag_bind_target_id_idx', ['target_id'], unique=False) + + op.create_table('tags', + sa.Column('id', postgresql.UUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('tenant_id', postgresql.UUID(), nullable=True), + sa.Column('type', sa.String(length=16), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('created_by', postgresql.UUID(), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.PrimaryKeyConstraint('id', name='tag_pkey') + ) + with op.batch_alter_table('tags', schema=None) as batch_op: + batch_op.create_index('tag_name_idx', ['name'], unique=False) + batch_op.create_index('tag_type_idx', ['type'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('tags', schema=None) as batch_op: + batch_op.drop_index('tag_type_idx') + batch_op.drop_index('tag_name_idx') + + op.drop_table('tags') + with op.batch_alter_table('tag_bindings', schema=None) as batch_op: + batch_op.drop_index('tag_bind_target_id_idx') + batch_op.drop_index('tag_bind_tag_id_idx') + + op.drop_table('tag_bindings') + # ### end Alembic commands ### diff --git a/api/models/dataset.py b/api/models/dataset.py index 5a893ad009..0e85008615 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -9,7 +9,7 @@ from sqlalchemy.dialects.postgresql import JSONB, UUID from extensions.ext_database import db from extensions.ext_storage import storage from models.account import Account -from models.model import App, UploadFile +from models.model import App, Tag, TagBinding, UploadFile class Dataset(db.Model): @@ -118,6 +118,20 @@ class Dataset(db.Model): } return self.retrieval_model if self.retrieval_model else default_retrieval_model + @property + def tags(self): + tags = db.session.query(Tag).join( + TagBinding, + Tag.id == TagBinding.tag_id + ).filter( + TagBinding.target_id == self.id, + TagBinding.tenant_id == self.tenant_id, + Tag.tenant_id == self.tenant_id, + Tag.type == 'knowledge' + ).all() + + return tags if tags else [] + @staticmethod def gen_collection_name_by_id(dataset_id: str) -> str: normalized_dataset_id = dataset_id.replace("-", "_") diff --git a/api/models/model.py b/api/models/model.py index df858e5219..9d5a492277 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -148,7 +148,7 @@ class App(db.Model): return [] agent_mode = app_model_config.agent_mode_dict tools = agent_mode.get('tools', []) - + provider_ids = [] for tool in tools: @@ -185,6 +185,20 @@ class App(db.Model): return deleted_tools + @property + def tags(self): + tags = db.session.query(Tag).join( + TagBinding, + Tag.id == TagBinding.tag_id + ).filter( + TagBinding.target_id == self.id, + TagBinding.tenant_id == self.tenant_id, + Tag.tenant_id == self.tenant_id, + Tag.type == 'app' + ).all() + + return tags if tags else [] + class AppModelConfig(db.Model): __tablename__ = 'app_model_configs' @@ -292,7 +306,8 @@ class AppModelConfig(db.Model): @property def agent_mode_dict(self) -> dict: - return json.loads(self.agent_mode) if self.agent_mode else {"enabled": False, "strategy": None, "tools": [], "prompt": None} + return json.loads(self.agent_mode) if self.agent_mode else {"enabled": False, "strategy": None, "tools": [], + "prompt": None} @property def chat_prompt_config_dict(self) -> dict: @@ -463,6 +478,7 @@ class InstalledApp(db.Model): return tenant + class Conversation(db.Model): __tablename__ = 'conversations' __table_args__ = ( @@ -1175,11 +1191,11 @@ class MessageAgentThought(db.Model): return json.loads(self.message_files) else: return [] - + @property def tools(self) -> list[str]: return self.tool.split(";") if self.tool else [] - + @property def tool_labels(self) -> dict: try: @@ -1189,7 +1205,7 @@ class MessageAgentThought(db.Model): return {} except Exception as e: return {} - + @property def tool_meta(self) -> dict: try: @@ -1199,7 +1215,7 @@ class MessageAgentThought(db.Model): return {} except Exception as e: return {} - + @property def tool_inputs_dict(self) -> dict: tools = self.tools @@ -1222,7 +1238,7 @@ class MessageAgentThought(db.Model): } except Exception as e: return {} - + @property def tool_outputs_dict(self) -> dict: tools = self.tools @@ -1249,6 +1265,7 @@ class MessageAgentThought(db.Model): tool: self.observation for tool in tools } + class DatasetRetrieverResource(db.Model): __tablename__ = 'dataset_retriever_resources' __table_args__ = ( @@ -1274,3 +1291,37 @@ class DatasetRetrieverResource(db.Model): retriever_from = db.Column(db.Text, nullable=False) created_by = db.Column(UUID, nullable=False) created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.current_timestamp()) + + +class Tag(db.Model): + __tablename__ = 'tags' + __table_args__ = ( + db.PrimaryKeyConstraint('id', name='tag_pkey'), + db.Index('tag_type_idx', 'type'), + db.Index('tag_name_idx', 'name'), + ) + + TAG_TYPE_LIST = ['knowledge', 'app'] + + id = db.Column(UUID, server_default=db.text('uuid_generate_v4()')) + tenant_id = db.Column(UUID, nullable=True) + type = db.Column(db.String(16), nullable=False) + name = db.Column(db.String(255), nullable=False) + created_by = db.Column(UUID, nullable=False) + created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + + +class TagBinding(db.Model): + __tablename__ = 'tag_bindings' + __table_args__ = ( + db.PrimaryKeyConstraint('id', name='tag_binding_pkey'), + db.Index('tag_bind_target_id_idx', 'target_id'), + db.Index('tag_bind_tag_id_idx', 'tag_id'), + ) + + id = db.Column(UUID, server_default=db.text('uuid_generate_v4()')) + tenant_id = db.Column(UUID, nullable=True) + tag_id = db.Column(UUID, nullable=True) + target_id = db.Column(UUID, nullable=True) + created_by = db.Column(UUID, nullable=False) + created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) diff --git a/api/services/app_service.py b/api/services/app_service.py index f57da12cf8..11073af09e 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -21,11 +21,12 @@ from extensions.ext_database import db from models.account import Account from models.model import App, AppMode, AppModelConfig from models.tools import ApiToolProvider +from services.tag_service import TagService from services.workflow_service import WorkflowService class AppService: - def get_paginate_apps(self, tenant_id: str, args: dict) -> Pagination: + def get_paginate_apps(self, tenant_id: str, args: dict) -> Pagination | None: """ Get app list with pagination :param tenant_id: tenant id @@ -49,6 +50,14 @@ class AppService: if 'name' in args and args['name']: name = args['name'][:30] filters.append(App.name.ilike(f'%{name}%')) + if 'tag_ids' in args and args['tag_ids']: + target_ids = TagService.get_target_ids_by_tag_ids('app', + tenant_id, + args['tag_ids']) + if target_ids: + filters.append(App.id.in_(target_ids)) + else: + return None app_models = db.paginate( db.select(App).where(*filters).order_by(App.created_at.desc()), diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index fe95a22cac..aaf6076d12 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -38,28 +38,39 @@ from services.errors.dataset import DatasetNameDuplicateError from services.errors.document import DocumentIndexingError from services.errors.file import FileNotExistsError from services.feature_service import FeatureModel, FeatureService +from services.tag_service import TagService from services.vector_service import VectorService from tasks.clean_notion_document_task import clean_notion_document_task from tasks.deal_dataset_vector_index_task import deal_dataset_vector_index_task from tasks.delete_segment_from_index_task import delete_segment_from_index_task from tasks.document_indexing_task import document_indexing_task from tasks.document_indexing_update_task import document_indexing_update_task +from tasks.duplicate_document_indexing_task import duplicate_document_indexing_task from tasks.recover_document_indexing_task import recover_document_indexing_task +from tasks.retry_document_indexing_task import retry_document_indexing_task class DatasetService: @staticmethod - def get_datasets(page, per_page, provider="vendor", tenant_id=None, user=None): + def get_datasets(page, per_page, provider="vendor", tenant_id=None, user=None, search=None, tag_ids=None): if user: permission_filter = db.or_(Dataset.created_by == user.id, Dataset.permission == 'all_team_members') else: permission_filter = Dataset.permission == 'all_team_members' - datasets = Dataset.query.filter( + query = Dataset.query.filter( db.and_(Dataset.provider == provider, Dataset.tenant_id == tenant_id, permission_filter)) \ - .order_by(Dataset.created_at.desc()) \ - .paginate( + .order_by(Dataset.created_at.desc()) + if search: + query = query.filter(db.and_(Dataset.name.ilike(f'%{search}%'))) + if tag_ids: + target_ids = TagService.get_target_ids_by_tag_ids('knowledge', tenant_id, tag_ids) + if target_ids: + query = query.filter(db.and_(Dataset.id.in_(target_ids))) + else: + return [], 0 + datasets = query.paginate( page=page, per_page=per_page, max_per_page=100, @@ -165,9 +176,36 @@ class DatasetService: # get embedding model setting try: model_manager = ModelManager() - embedding_model = model_manager.get_default_model_instance( + embedding_model = model_manager.get_model_instance( tenant_id=current_user.current_tenant_id, - model_type=ModelType.TEXT_EMBEDDING + provider=data['embedding_model_provider'], + model_type=ModelType.TEXT_EMBEDDING, + model=data['embedding_model'] + ) + filtered_data['embedding_model'] = embedding_model.model + filtered_data['embedding_model_provider'] = embedding_model.provider + dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + embedding_model.provider, + embedding_model.model + ) + filtered_data['collection_binding_id'] = dataset_collection_binding.id + except LLMBadRequestError: + raise ValueError( + "No Embedding Model available. Please configure a valid provider " + "in the Settings -> Model Provider.") + except ProviderTokenNotInitError as ex: + raise ValueError(ex.description) + else: + if data['embedding_model_provider'] != dataset.embedding_model_provider or \ + data['embedding_model'] != dataset.embedding_model: + action = 'update' + try: + model_manager = ModelManager() + embedding_model = model_manager.get_model_instance( + tenant_id=current_user.current_tenant_id, + provider=data['embedding_model_provider'], + model_type=ModelType.TEXT_EMBEDDING, + model=data['embedding_model'] ) filtered_data['embedding_model'] = embedding_model.model filtered_data['embedding_model_provider'] = embedding_model.provider @@ -376,6 +414,15 @@ class DocumentService: return documents + @staticmethod + def get_error_documents_by_dataset_id(dataset_id: str) -> list[Document]: + documents = db.session.query(Document).filter( + Document.dataset_id == dataset_id, + Document.indexing_status == 'error' or Document.indexing_status == 'paused' + ).all() + + return documents + @staticmethod def get_batch_documents(dataset_id: str, batch: str) -> list[Document]: documents = db.session.query(Document).filter( @@ -440,6 +487,20 @@ class DocumentService: # trigger async task recover_document_indexing_task.delay(document.dataset_id, document.id) + @staticmethod + def retry_document(dataset_id: str, documents: list[Document]): + for document in documents: + # retry document indexing + document.indexing_status = 'waiting' + db.session.add(document) + db.session.commit() + # add retry flag + retry_indexing_cache_key = 'document_{}_is_retried'.format(document.id) + redis_client.setex(retry_indexing_cache_key, 600, 1) + # trigger async task + document_ids = [document.id for document in documents] + retry_document_indexing_task.delay(dataset_id, document_ids) + @staticmethod def get_documents_position(dataset_id): document = Document.query.filter_by(dataset_id=dataset_id).order_by(Document.position.desc()).first() @@ -537,6 +598,7 @@ class DocumentService: db.session.commit() position = DocumentService.get_documents_position(dataset.id) document_ids = [] + duplicate_document_ids = [] if document_data["data_source"]["type"] == "upload_file": upload_file_list = document_data["data_source"]["info_list"]['file_info_list']['file_ids'] for file_id in upload_file_list: @@ -553,6 +615,28 @@ class DocumentService: data_source_info = { "upload_file_id": file_id, } + # check duplicate + if document_data.get('duplicate', False): + document = Document.query.filter_by( + dataset_id=dataset.id, + tenant_id=current_user.current_tenant_id, + data_source_type='upload_file', + enabled=True, + name=file_name + ).first() + if document: + document.dataset_process_rule_id = dataset_process_rule.id + document.updated_at = datetime.datetime.utcnow() + document.created_from = created_from + document.doc_form = document_data['doc_form'] + document.doc_language = document_data['doc_language'] + document.data_source_info = json.dumps(data_source_info) + document.batch = batch + document.indexing_status = 'waiting' + db.session.add(document) + documents.append(document) + duplicate_document_ids.append(document.id) + continue document = DocumentService.build_document(dataset, dataset_process_rule.id, document_data["data_source"]["type"], document_data["doc_form"], @@ -618,7 +702,10 @@ class DocumentService: db.session.commit() # trigger async task - document_indexing_task.delay(dataset.id, document_ids) + if document_ids: + document_indexing_task.delay(dataset.id, document_ids) + if duplicate_document_ids: + duplicate_document_indexing_task.delay(dataset.id, duplicate_document_ids) return documents, batch @@ -626,7 +713,8 @@ class DocumentService: def check_documents_upload_quota(count: int, features: FeatureModel): can_upload_size = features.documents_upload_quota.limit - features.documents_upload_quota.size if count > can_upload_size: - raise ValueError(f'You have reached the limit of your subscription. Only {can_upload_size} documents can be uploaded.') + raise ValueError( + f'You have reached the limit of your subscription. Only {can_upload_size} documents can be uploaded.') @staticmethod def build_document(dataset: Dataset, process_rule_id: str, data_source_type: str, document_form: str, @@ -752,7 +840,6 @@ class DocumentService: db.session.commit() # trigger async task document_indexing_update_task.delay(document.dataset_id, document.id) - return document @staticmethod diff --git a/api/services/tag_service.py b/api/services/tag_service.py new file mode 100644 index 0000000000..d6eba38fbd --- /dev/null +++ b/api/services/tag_service.py @@ -0,0 +1,161 @@ +import uuid + +from flask_login import current_user +from sqlalchemy import func +from werkzeug.exceptions import NotFound + +from extensions.ext_database import db +from models.dataset import Dataset +from models.model import App, Tag, TagBinding + + +class TagService: + @staticmethod + def get_tags(tag_type: str, current_tenant_id: str, keyword: str = None) -> list: + query = db.session.query( + Tag.id, Tag.type, Tag.name, func.count(TagBinding.id).label('binding_count') + ).outerjoin( + TagBinding, Tag.id == TagBinding.tag_id + ).filter( + Tag.type == tag_type, + Tag.tenant_id == current_tenant_id + ) + if keyword: + query = query.filter(db.and_(Tag.name.ilike(f'%{keyword}%'))) + query = query.group_by( + Tag.id + ) + results = query.order_by(Tag.created_at.desc()).all() + return results + + @staticmethod + def get_target_ids_by_tag_ids(tag_type: str, current_tenant_id: str, tag_ids: list) -> list: + tags = db.session.query(Tag).filter( + Tag.id.in_(tag_ids), + Tag.tenant_id == current_tenant_id, + Tag.type == tag_type + ).all() + if not tags: + return [] + tag_ids = [tag.id for tag in tags] + tag_bindings = db.session.query( + TagBinding.target_id + ).filter( + TagBinding.tag_id.in_(tag_ids), + TagBinding.tenant_id == current_tenant_id + ).all() + if not tag_bindings: + return [] + results = [tag_binding.target_id for tag_binding in tag_bindings] + return results + + @staticmethod + def get_tags_by_target_id(tag_type: str, current_tenant_id: str, target_id: str) -> list: + tags = db.session.query(Tag).join( + TagBinding, + Tag.id == TagBinding.tag_id + ).filter( + TagBinding.target_id == target_id, + TagBinding.tenant_id == current_tenant_id, + Tag.tenant_id == current_tenant_id, + Tag.type == tag_type + ).all() + + return tags if tags else [] + + + @staticmethod + def save_tags(args: dict) -> Tag: + tag = Tag( + id=str(uuid.uuid4()), + name=args['name'], + type=args['type'], + created_by=current_user.id, + tenant_id=current_user.current_tenant_id + ) + db.session.add(tag) + db.session.commit() + return tag + + @staticmethod + def update_tags(args: dict, tag_id: str) -> Tag: + tag = db.session.query(Tag).filter(Tag.id == tag_id).first() + if not tag: + raise NotFound("Tag not found") + tag.name = args['name'] + db.session.commit() + return tag + + @staticmethod + def get_tag_binding_count(tag_id: str) -> int: + count = db.session.query(TagBinding).filter(TagBinding.tag_id == tag_id).count() + return count + + @staticmethod + def delete_tag(tag_id: str): + tag = db.session.query(Tag).filter(Tag.id == tag_id).first() + if not tag: + raise NotFound("Tag not found") + db.session.delete(tag) + # delete tag binding + tag_bindings = db.session.query(TagBinding).filter(TagBinding.tag_id == tag_id).all() + if tag_bindings: + for tag_binding in tag_bindings: + db.session.delete(tag_binding) + db.session.commit() + + @staticmethod + def save_tag_binding(args): + # check if target exists + TagService.check_target_exists(args['type'], args['target_id']) + # save tag binding + for tag_id in args['tag_ids']: + tag_binding = db.session.query(TagBinding).filter( + TagBinding.tag_id == tag_id, + TagBinding.target_id == args['target_id'] + ).first() + if tag_binding: + continue + new_tag_binding = TagBinding( + tag_id=tag_id, + target_id=args['target_id'], + tenant_id=current_user.current_tenant_id, + created_by=current_user.id + ) + db.session.add(new_tag_binding) + db.session.commit() + + @staticmethod + def delete_tag_binding(args): + # check if target exists + TagService.check_target_exists(args['type'], args['target_id']) + # delete tag binding + tag_bindings = db.session.query(TagBinding).filter( + TagBinding.target_id == args['target_id'], + TagBinding.tag_id == (args['tag_id']) + ).first() + if tag_bindings: + db.session.delete(tag_bindings) + db.session.commit() + + + + @staticmethod + def check_target_exists(type: str, target_id: str): + if type == 'knowledge': + dataset = db.session.query(Dataset).filter( + Dataset.tenant_id == current_user.current_tenant_id, + Dataset.id == target_id + ).first() + if not dataset: + raise NotFound("Dataset not found") + elif type == 'app': + app = db.session.query(App).filter( + App.tenant_id == current_user.current_tenant_id, + App.id == target_id + ).first() + if not app: + raise NotFound("App not found") + else: + raise NotFound("Invalid binding type") + diff --git a/api/tasks/clean_dataset_task.py b/api/tasks/clean_dataset_task.py index 74d74ddf46..4de587d26a 100644 --- a/api/tasks/clean_dataset_task.py +++ b/api/tasks/clean_dataset_task.py @@ -16,6 +16,7 @@ from models.dataset import ( ) +# Add import statement for ValueError @shared_task(queue='dataset') def clean_dataset_task(dataset_id: str, tenant_id: str, indexing_technique: str, index_struct: str, collection_binding_id: str, doc_form: str): @@ -48,6 +49,9 @@ def clean_dataset_task(dataset_id: str, tenant_id: str, indexing_technique: str, logging.info(click.style('No documents found for dataset: {}'.format(dataset_id), fg='green')) else: logging.info(click.style('Cleaning documents for dataset: {}'.format(dataset_id), fg='green')) + # Specify the index type before initializing the index processor + if doc_form is None: + raise ValueError("Index type must be specified.") index_processor = IndexProcessorFactory(doc_form).init_index_processor() index_processor.clean(dataset, None) diff --git a/api/tasks/deal_dataset_vector_index_task.py b/api/tasks/deal_dataset_vector_index_task.py index 3827d62fbf..c1b0e7f1a4 100644 --- a/api/tasks/deal_dataset_vector_index_task.py +++ b/api/tasks/deal_dataset_vector_index_task.py @@ -64,6 +64,39 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str): # save vector index index_processor.load(dataset, documents, with_keywords=False) + elif action == 'update': + # clean index + index_processor.clean(dataset, None, with_keywords=False) + dataset_documents = db.session.query(DatasetDocument).filter( + DatasetDocument.dataset_id == dataset_id, + DatasetDocument.indexing_status == 'completed', + DatasetDocument.enabled == True, + DatasetDocument.archived == False, + ).all() + # add new index + if dataset_documents: + documents = [] + for dataset_document in dataset_documents: + # delete from vector index + segments = db.session.query(DocumentSegment).filter( + DocumentSegment.document_id == dataset_document.id, + DocumentSegment.enabled == True + ).order_by(DocumentSegment.position.asc()).all() + for segment in segments: + document = Document( + page_content=segment.content, + metadata={ + "doc_id": segment.index_node_id, + "doc_hash": segment.index_node_hash, + "document_id": segment.document_id, + "dataset_id": segment.dataset_id, + } + ) + + documents.append(document) + + # save vector index + index_processor.load(dataset, documents, with_keywords=False) end_at = time.perf_counter() logging.info( diff --git a/api/tasks/duplicate_document_indexing_task.py b/api/tasks/duplicate_document_indexing_task.py new file mode 100644 index 0000000000..1854589e7f --- /dev/null +++ b/api/tasks/duplicate_document_indexing_task.py @@ -0,0 +1,94 @@ +import datetime +import logging +import time + +import click +from celery import shared_task +from flask import current_app + +from core.indexing_runner import DocumentIsPausedException, IndexingRunner +from core.rag.index_processor.index_processor_factory import IndexProcessorFactory +from extensions.ext_database import db +from models.dataset import Dataset, Document, DocumentSegment +from services.feature_service import FeatureService + + +@shared_task(queue='dataset') +def duplicate_document_indexing_task(dataset_id: str, document_ids: list): + """ + Async process document + :param dataset_id: + :param document_ids: + + Usage: duplicate_document_indexing_task.delay(dataset_id, document_id) + """ + documents = [] + start_at = time.perf_counter() + + dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first() + + # check document limit + features = FeatureService.get_features(dataset.tenant_id) + try: + if features.billing.enabled: + vector_space = features.vector_space + count = len(document_ids) + batch_upload_limit = int(current_app.config['BATCH_UPLOAD_LIMIT']) + if count > batch_upload_limit: + raise ValueError(f"You have reached the batch upload limit of {batch_upload_limit}.") + if 0 < vector_space.limit <= vector_space.size: + raise ValueError("Your total number of documents plus the number of uploads have over the limit of " + "your subscription.") + except Exception as e: + for document_id in document_ids: + document = db.session.query(Document).filter( + Document.id == document_id, + Document.dataset_id == dataset_id + ).first() + if document: + document.indexing_status = 'error' + document.error = str(e) + document.stopped_at = datetime.datetime.utcnow() + db.session.add(document) + db.session.commit() + return + + for document_id in document_ids: + logging.info(click.style('Start process document: {}'.format(document_id), fg='green')) + + document = db.session.query(Document).filter( + Document.id == document_id, + Document.dataset_id == dataset_id + ).first() + + if document: + # clean old data + index_type = document.doc_form + index_processor = IndexProcessorFactory(index_type).init_index_processor() + + segments = db.session.query(DocumentSegment).filter(DocumentSegment.document_id == document_id).all() + if segments: + index_node_ids = [segment.index_node_id for segment in segments] + + # delete from vector index + index_processor.clean(dataset, index_node_ids) + + for segment in segments: + db.session.delete(segment) + db.session.commit() + + document.indexing_status = 'parsing' + document.processing_started_at = datetime.datetime.utcnow() + documents.append(document) + db.session.add(document) + db.session.commit() + + try: + indexing_runner = IndexingRunner() + indexing_runner.run(documents) + end_at = time.perf_counter() + logging.info(click.style('Processed dataset: {} latency: {}'.format(dataset_id, end_at - start_at), fg='green')) + except DocumentIsPausedException as ex: + logging.info(click.style(str(ex), fg='yellow')) + except Exception: + pass diff --git a/api/tasks/retry_document_indexing_task.py b/api/tasks/retry_document_indexing_task.py new file mode 100644 index 0000000000..1114809b30 --- /dev/null +++ b/api/tasks/retry_document_indexing_task.py @@ -0,0 +1,91 @@ +import datetime +import logging +import time + +import click +from celery import shared_task + +from core.indexing_runner import IndexingRunner +from core.rag.index_processor.index_processor_factory import IndexProcessorFactory +from extensions.ext_database import db +from extensions.ext_redis import redis_client +from models.dataset import Dataset, Document, DocumentSegment +from services.feature_service import FeatureService + + +@shared_task(queue='dataset') +def retry_document_indexing_task(dataset_id: str, document_ids: list[str]): + """ + Async process document + :param dataset_id: + :param document_ids: + + Usage: retry_document_indexing_task.delay(dataset_id, document_id) + """ + documents = [] + start_at = time.perf_counter() + + dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first() + for document_id in document_ids: + retry_indexing_cache_key = 'document_{}_is_retried'.format(document_id) + # check document limit + features = FeatureService.get_features(dataset.tenant_id) + try: + if features.billing.enabled: + vector_space = features.vector_space + if 0 < vector_space.limit <= vector_space.size: + raise ValueError("Your total number of documents plus the number of uploads have over the limit of " + "your subscription.") + except Exception as e: + document = db.session.query(Document).filter( + Document.id == document_id, + Document.dataset_id == dataset_id + ).first() + if document: + document.indexing_status = 'error' + document.error = str(e) + document.stopped_at = datetime.datetime.utcnow() + db.session.add(document) + db.session.commit() + redis_client.delete(retry_indexing_cache_key) + return + + logging.info(click.style('Start retry document: {}'.format(document_id), fg='green')) + document = db.session.query(Document).filter( + Document.id == document_id, + Document.dataset_id == dataset_id + ).first() + try: + if document: + # clean old data + index_processor = IndexProcessorFactory(document.doc_form).init_index_processor() + + segments = db.session.query(DocumentSegment).filter(DocumentSegment.document_id == document_id).all() + if segments: + index_node_ids = [segment.index_node_id for segment in segments] + # delete from vector index + index_processor.clean(dataset, index_node_ids) + + for segment in segments: + db.session.delete(segment) + db.session.commit() + + document.indexing_status = 'parsing' + document.processing_started_at = datetime.datetime.utcnow() + db.session.add(document) + db.session.commit() + + indexing_runner = IndexingRunner() + indexing_runner.run([document]) + redis_client.delete(retry_indexing_cache_key) + except Exception as ex: + document.indexing_status = 'error' + document.error = str(ex) + document.stopped_at = datetime.datetime.utcnow() + db.session.add(document) + db.session.commit() + logging.info(click.style(str(ex), fg='yellow')) + redis_client.delete(retry_indexing_cache_key) + pass + end_at = time.perf_counter() + logging.info(click.style('Retry dataset: {} latency: {}'.format(dataset_id, end_at - start_at), fg='green')) diff --git a/web/app/(commonLayout)/apps/AppCard.tsx b/web/app/(commonLayout)/apps/AppCard.tsx index 1103df3f64..4572d7e8d4 100644 --- a/web/app/(commonLayout)/apps/AppCard.tsx +++ b/web/app/(commonLayout)/apps/AppCard.tsx @@ -2,7 +2,7 @@ import { useContext, useContextSelector } from 'use-context-selector' import { useRouter } from 'next/navigation' -import { useCallback, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' import s from './style.module.css' @@ -22,9 +22,12 @@ import { useProviderContext } from '@/context/provider-context' import { NEED_REFRESH_APP_LIST_KEY } from '@/config' import { AiText, ChatBot, CuteRobote } from '@/app/components/base/icons/src/vender/solid/communication' import { Route } from '@/app/components/base/icons/src/vender/solid/mapsAndTravel' +import { DotsHorizontal } from '@/app/components/base/icons/src/vender/line/general' import type { CreateAppModalProps } from '@/app/components/explore/create-app-modal' import EditAppModal from '@/app/components/explore/create-app-modal' import SwitchAppModal from '@/app/components/app/switch-app-modal' +import type { Tag } from '@/app/components/base/tag-management/constant' +import TagSelector from '@/app/components/base/tag-management/selector' export type AppCardProps = { app: App @@ -142,6 +145,9 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { } const Operations = (props: HtmlContentProps) => { + const onMouseLeave = async () => { + props.onClose?.() + } const onClickSettings = async (e: React.MouseEvent) => { e.stopPropagation() props.onClick?.() @@ -173,7 +179,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { setShowConfirmDelete(true) } return ( -
+
@@ -208,6 +214,11 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { ) } + const [tags, setTags] = useState(app.tags) + useEffect(() => { + setTags(app.tags) + }, [app.tags]) + return ( <>
{ {app.mode === 'completion' &&
{t('app.types.completion').toUpperCase()}
}
- {isCurrentWorkspaceManager && } - position="br" - trigger="click" - btnElement={
} - btnClassName={open => - cn( - open ? '!bg-gray-100 !shadow-none' : '!bg-transparent', - '!hidden h-8 w-8 !p-2 rounded-md border-none hover:!bg-gray-100 group-hover:!inline-flex', - ) - } - className={'!w-[128px] h-fit !z-20'} - popupClassName={ - (app.mode === 'completion' || app.mode === 'chat') - ? '!w-[238px] translate-x-[-110px]' - : '' - } - manualClose - />}
-
{app.description}
+
+ {app.description} +
+
+
{ + e.stopPropagation() + e.preventDefault() + }}> +
+ tag.id)} + selectedTags={tags} + onCacheUpdate={setTags} + onChange={onRefresh} + /> +
+
+ {isCurrentWorkspaceManager && ( + <> +
+
+ } + position="br" + trigger="click" + btnElement={ +
+ +
+ } + btnClassName={open => + cn( + open ? '!bg-black/5 !shadow-none' : '!bg-transparent', + 'h-8 w-8 !p-2 rounded-md border-none hover:!bg-black/5', + ) + } + popupClassName={ + (app.mode === 'completion' || app.mode === 'chat') + ? '!w-[238px] translate-x-[-110px]' + : '' + } + className={'!w-[128px] h-fit !z-20'} + /> +
+ + )} +
{showEditModal && ( { if (!pageIndex || previousPageData.has_more) { @@ -36,6 +39,9 @@ const getKey = ( else delete params.params.mode + if (tags.length) + params.params.tag_ids = tags + return params } return null @@ -44,14 +50,17 @@ const getKey = ( const Apps = () => { const { t } = useTranslation() const { isCurrentWorkspaceManager } = useAppContext() + const showTagManagementModal = useTagStore(s => s.showTagManagementModal) const [activeTab, setActiveTab] = useTabSearchParams({ defaultTab: 'all', }) + const [tagFilterValue, setTagFilterValue] = useState([]) + const [tagIDs, setTagIDs] = useState([]) const [keywords, setKeywords] = useState('') const [searchKeywords, setSearchKeywords] = useState('') const { data, isLoading, setSize, mutate } = useSWRInfinite( - (pageIndex: number, previousPageData: AppListResponse) => getKey(pageIndex, previousPageData, activeTab, searchKeywords), + (pageIndex: number, previousPageData: AppListResponse) => getKey(pageIndex, previousPageData, activeTab, tagIDs, searchKeywords), fetchAppList, { revalidateFirstPage: true }, ) @@ -61,7 +70,6 @@ const Apps = () => { { value: 'all', text: t('app.types.all'), icon: }, { value: 'chat', text: t('app.types.chatbot'), icon: }, { value: 'agent-chat', text: t('app.types.agent'), icon: }, - // { value: 'completion', text: t('app.newApp.completeApp'), icon: }, { value: 'workflow', text: t('app.types.workflow'), icon: }, ] @@ -88,14 +96,17 @@ const Apps = () => { const { run: handleSearch } = useDebounceFn(() => { setSearchKeywords(keywords) }, { wait: 500 }) - const handleKeywordsChange = (value: string) => { setKeywords(value) handleSearch() } - const handleClear = () => { - handleKeywordsChange('') + const { run: handleTagsUpdate } = useDebounceFn(() => { + setTagIDs(tagFilterValue) + }, { wait: 500 }) + const handleTagsChange = (value: string[]) => { + setTagFilterValue(value) + handleTagsUpdate() } return ( @@ -106,31 +117,9 @@ const Apps = () => { onChange={setActiveTab} options={options} /> -
-
-
- { - handleKeywordsChange(e.target.value) - }} - autoComplete="off" - /> - { - keywords && ( -
- -
- ) - } +
+ +
+ {showTagManagementModal && ( + + )} ) } diff --git a/web/app/(commonLayout)/datasets/Container.tsx b/web/app/(commonLayout)/datasets/Container.tsx index 5c2f227222..7e3a253797 100644 --- a/web/app/(commonLayout)/datasets/Container.tsx +++ b/web/app/(commonLayout)/datasets/Container.tsx @@ -1,8 +1,9 @@ 'use client' // Libraries -import { useRef } from 'react' +import { useRef, useState } from 'react' import { useTranslation } from 'react-i18next' +import { useDebounceFn } from 'ahooks' import useSWR from 'swr' // Components @@ -11,15 +12,20 @@ import DatasetFooter from './DatasetFooter' import ApiServer from './ApiServer' import Doc from './Doc' import TabSliderNew from '@/app/components/base/tab-slider-new' +import SearchInput from '@/app/components/base/search-input' +import TagManagementModal from '@/app/components/base/tag-management' +import TagFilter from '@/app/components/base/tag-management/filter' // Services import { fetchDatasetApiBaseUrl } from '@/service/datasets' // Hooks import { useTabSearchParams } from '@/hooks/use-tab-searchparams' +import { useStore as useTagStore } from '@/app/components/base/tag-management/store' const Container = () => { const { t } = useTranslation() + const showTagManagementModal = useTagStore(s => s.showTagManagementModal) const options = [ { value: 'dataset', text: t('dataset.datasets') }, @@ -32,6 +38,25 @@ const Container = () => { const containerRef = useRef(null) const { data } = useSWR(activeTab === 'dataset' ? null : '/datasets/api-base-info', fetchDatasetApiBaseUrl) + const [keywords, setKeywords] = useState('') + const [searchKeywords, setSearchKeywords] = useState('') + const { run: handleSearch } = useDebounceFn(() => { + setSearchKeywords(keywords) + }, { wait: 500 }) + const handleKeywordsChange = (value: string) => { + setKeywords(value) + handleSearch() + } + const [tagFilterValue, setTagFilterValue] = useState([]) + const [tagIDs, setTagIDs] = useState([]) + const { run: handleTagsUpdate } = useDebounceFn(() => { + setTagIDs(tagFilterValue) + }, { wait: 500 }) + const handleTagsChange = (value: string[]) => { + setTagFilterValue(value) + handleTagsUpdate() + } + return (
@@ -40,13 +65,22 @@ const Container = () => { onChange={newActiveTab => setActiveTab(newActiveTab)} options={options} /> + {activeTab === 'dataset' && ( +
+ + +
+ )} {activeTab === 'api' && data && }
{activeTab === 'dataset' && ( <> - + + {showTagManagementModal && ( + + )} )} diff --git a/web/app/(commonLayout)/datasets/DatasetCard.tsx b/web/app/(commonLayout)/datasets/DatasetCard.tsx index ffcf9a6c0c..9fe0af4bba 100644 --- a/web/app/(commonLayout)/datasets/DatasetCard.tsx +++ b/web/app/(commonLayout)/datasets/DatasetCard.tsx @@ -2,41 +2,44 @@ import { useContext } from 'use-context-selector' import Link from 'next/link' -import type { MouseEventHandler } from 'react' -import { useCallback, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' -import style from '../list.module.css' import Confirm from '@/app/components/base/confirm' import { ToastContext } from '@/app/components/base/toast' import { deleteDataset } from '@/service/datasets' -import AppIcon from '@/app/components/base/app-icon' import type { DataSet } from '@/models/datasets' import Tooltip from '@/app/components/base/tooltip' +import { Folder } from '@/app/components/base/icons/src/vender/solid/files' +import type { HtmlContentProps } from '@/app/components/base/popover' +import CustomPopover from '@/app/components/base/popover' +import Divider from '@/app/components/base/divider' +import { DotsHorizontal } from '@/app/components/base/icons/src/vender/line/general' +import RenameDatasetModal from '@/app/components/datasets/rename-modal' +import type { Tag } from '@/app/components/base/tag-management/constant' +import TagSelector from '@/app/components/base/tag-management/selector' export type DatasetCardProps = { dataset: DataSet - onDelete?: () => void + onSuccess?: () => void } const DatasetCard = ({ dataset, - onDelete, + onSuccess, }: DatasetCardProps) => { const { t } = useTranslation() const { notify } = useContext(ToastContext) + const [tags, setTags] = useState(dataset.tags) + const [showRenameModal, setShowRenameModal] = useState(false) const [showConfirmDelete, setShowConfirmDelete] = useState(false) - const onDeleteClick: MouseEventHandler = useCallback((e) => { - e.preventDefault() - setShowConfirmDelete(true) - }, []) const onConfirmDelete = useCallback(async () => { try { await deleteDataset(dataset.id) notify({ type: 'success', message: t('dataset.datasetDeleted') }) - if (onDelete) - onDelete() + if (onSuccess) + onSuccess() } catch (e: any) { notify({ type: 'error', message: `${t('dataset.datasetDeleteFailed')}${'message' in e ? `: ${e.message}` : ''}` }) @@ -44,53 +47,158 @@ const DatasetCard = ({ setShowConfirmDelete(false) }, [dataset.id]) + const Operations = (props: HtmlContentProps) => { + const onMouseLeave = async () => { + props.onClose?.() + } + const onClickRename = async (e: React.MouseEvent) => { + e.stopPropagation() + props.onClick?.() + e.preventDefault() + setShowRenameModal(true) + } + const onClickDelete = async (e: React.MouseEvent) => { + e.stopPropagation() + props.onClick?.() + e.preventDefault() + setShowConfirmDelete(true) + } + return ( +
+
+ {t('common.operation.settings')} +
+ +
+ + {t('common.operation.delete')} + +
+
+ ) + } + + useEffect(() => { + setTags(dataset.tags) + }, [dataset]) + return ( <> - -
- -
-
- {dataset.name} + +
+
+ +
+
+
+
{dataset.name}
+ {!dataset.embedding_available && ( + + {t('dataset.unavailable')} + + )} +
+
+
+ {dataset.document_count}{t('dataset.documentCount')} + · + {Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')} + · + {dataset.app_count}{t('dataset.appCount')} +
- {!dataset.embedding_available && ( - - {t('dataset.unavailable')} - +
+
+ title={dataset.description}> + {dataset.description}
-
{dataset.description}
-
- - - {dataset.document_count}{t('dataset.documentCount')} - - - - {Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')} - - - - {dataset.app_count}{t('dataset.appCount')} - +
+
{ + e.stopPropagation() + e.preventDefault() + }}> +
+ tag.id)} + selectedTags={tags} + onCacheUpdate={setTags} + onChange={onSuccess} + /> +
+
+
+
+ } + position="br" + trigger="click" + btnElement={ +
+ +
+ } + btnClassName={open => + cn( + open ? '!bg-black/5 !shadow-none' : '!bg-transparent', + 'h-8 w-8 !p-2 rounded-md border-none hover:!bg-black/5', + ) + } + className={'!w-[128px] h-fit !z-20'} + /> +
- - {showConfirmDelete && ( - setShowConfirmDelete(false)} - onConfirm={onConfirmDelete} - onCancel={() => setShowConfirmDelete(false)} - /> - )} + {showRenameModal && ( + setShowRenameModal(false)} + onSuccess={onSuccess} + /> + )} + {showConfirmDelete && ( + setShowConfirmDelete(false)} + onConfirm={onConfirmDelete} + onCancel={() => setShowConfirmDelete(false)} + /> + )} ) } diff --git a/web/app/(commonLayout)/datasets/Datasets.tsx b/web/app/(commonLayout)/datasets/Datasets.tsx index adaf378975..a80af2f023 100644 --- a/web/app/(commonLayout)/datasets/Datasets.tsx +++ b/web/app/(commonLayout)/datasets/Datasets.tsx @@ -10,21 +10,46 @@ import type { DataSetListResponse } from '@/models/datasets' import { fetchDatasets } from '@/service/datasets' import { useAppContext } from '@/context/app-context' -const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => { - if (!pageIndex || previousPageData.has_more) - return { url: 'datasets', params: { page: pageIndex + 1, limit: 30 } } +const getKey = ( + pageIndex: number, + previousPageData: DataSetListResponse, + tags: string[], + keyword: string, +) => { + if (!pageIndex || previousPageData.has_more) { + const params: any = { + url: 'datasets', + params: { + page: pageIndex + 1, + limit: 30, + }, + } + if (tags.length) + params.params.tag_ids = tags + if (keyword) + params.params.keyword = keyword + return params + } return null } type Props = { containerRef: React.RefObject + tags: string[] + keywords: string } const Datasets = ({ containerRef, + tags, + keywords, }: Props) => { const { isCurrentWorkspaceManager } = useAppContext() - const { data, isLoading, setSize, mutate } = useSWRInfinite(getKey, fetchDatasets, { revalidateFirstPage: false, revalidateAll: true }) + const { data, isLoading, setSize, mutate } = useSWRInfinite( + (pageIndex: number, previousPageData: DataSetListResponse) => getKey(pageIndex, previousPageData, tags, keywords), + fetchDatasets, + { revalidateFirstPage: false, revalidateAll: true }, + ) const loadingStateRef = useRef(false) const anchorRef = useRef(null) @@ -53,7 +78,7 @@ const Datasets = ({ ) diff --git a/web/app/(commonLayout)/datasets/NewDatasetCard.tsx b/web/app/(commonLayout)/datasets/NewDatasetCard.tsx index 079fd0e82b..5ce79adadc 100644 --- a/web/app/(commonLayout)/datasets/NewDatasetCard.tsx +++ b/web/app/(commonLayout)/datasets/NewDatasetCard.tsx @@ -1,27 +1,22 @@ 'use client' import { forwardRef } from 'react' -import classNames from 'classnames' import { useTranslation } from 'react-i18next' -import Link from 'next/link' -import style from '../list.module.css' +import { Plus } from '@/app/components/base/icons/src/vender/line/general' const CreateAppCard = forwardRef((_, ref) => { const { t } = useTranslation() return ( - -
- - - -
- {t('dataset.createDataset')} + +
+
+
+
{t('dataset.createDataset')}
-
{t('dataset.createDatasetIntro')}
- {/*
{t('app.createFromConfigFile')}
*/} - +
{t('dataset.createDatasetIntro')}
+ ) }) diff --git a/web/app/(commonLayout)/datasets/assets/application.svg b/web/app/(commonLayout)/datasets/assets/application.svg deleted file mode 100644 index 0384961f82..0000000000 --- a/web/app/(commonLayout)/datasets/assets/application.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/web/app/(commonLayout)/datasets/assets/doc.svg b/web/app/(commonLayout)/datasets/assets/doc.svg deleted file mode 100644 index 6bb150cfd6..0000000000 --- a/web/app/(commonLayout)/datasets/assets/doc.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/web/app/(commonLayout)/datasets/assets/text.svg b/web/app/(commonLayout)/datasets/assets/text.svg deleted file mode 100644 index 6bb150cfd6..0000000000 --- a/web/app/(commonLayout)/datasets/assets/text.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/web/app/(commonLayout)/list.module.css b/web/app/(commonLayout)/list.module.css index 1cadc9f81e..3e34f11ea5 100644 --- a/web/app/(commonLayout)/list.module.css +++ b/web/app/(commonLayout)/list.module.css @@ -159,18 +159,6 @@ background-image: url("./apps/assets/completion-solid.svg"); } -.docIcon { - background-image: url("./datasets/assets/doc.svg"); -} - -.textIcon { - background-image: url("./datasets/assets/text.svg"); -} - -.applicationIcon { - background-image: url("./datasets/assets/application.svg"); -} - .newItemCardHeading { @apply transition-colors duration-200 ease-in-out; } diff --git a/web/app/components/base/icons/assets/vender/line/financeAndECommerce/tag-01.svg b/web/app/components/base/icons/assets/vender/line/financeAndECommerce/tag-01.svg new file mode 100644 index 0000000000..286322dd3b --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/financeAndECommerce/tag-01.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/vender/line/financeAndECommerce/tag-03.svg b/web/app/components/base/icons/assets/vender/line/financeAndECommerce/tag-03.svg new file mode 100644 index 0000000000..a8eeebf262 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/financeAndECommerce/tag-03.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag01.json b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag01.json new file mode 100644 index 0000000000..b6f838d72f --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag01.json @@ -0,0 +1,66 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "14", + "height": "14", + "viewBox": "0 0 14 14", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Icon", + "clip-path": "url(#clip0_17795_9693)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Icon_2", + "d": "M4.66699 4.6665H4.67283M1.16699 3.03317L1.16699 5.6433C1.16699 5.92866 1.16699 6.07134 1.19923 6.20561C1.22781 6.32465 1.27495 6.43845 1.33891 6.54284C1.41106 6.66057 1.51195 6.76146 1.71373 6.96324L6.18709 11.4366C6.88012 12.1296 7.22664 12.4761 7.62621 12.606C7.97769 12.7202 8.35629 12.7202 8.70777 12.606C9.10735 12.4761 9.45386 12.1296 10.1469 11.4366L11.4371 10.1464C12.1301 9.45337 12.4766 9.10686 12.6065 8.70728C12.7207 8.35581 12.7207 7.9772 12.6065 7.62572C12.4766 7.22615 12.1301 6.87963 11.4371 6.1866L6.96372 1.71324C6.76195 1.51146 6.66106 1.41057 6.54332 1.33842C6.43894 1.27446 6.32514 1.22732 6.20609 1.19874C6.07183 1.1665 5.92915 1.1665 5.64379 1.1665L3.03366 1.1665C2.38026 1.1665 2.05357 1.1665 1.804 1.29366C1.58448 1.40552 1.406 1.58399 1.29415 1.80352C1.16699 2.05308 1.16699 2.37978 1.16699 3.03317ZM4.95866 4.6665C4.95866 4.82759 4.82808 4.95817 4.66699 4.95817C4.50591 4.95817 4.37533 4.82759 4.37533 4.6665C4.37533 4.50542 4.50591 4.37484 4.66699 4.37484C4.82808 4.37484 4.95866 4.50542 4.95866 4.6665Z", + "stroke": "currentColor", + "stroke-width": "1.25", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "defs", + "attributes": {}, + "children": [ + { + "type": "element", + "name": "clipPath", + "attributes": { + "id": "clip0_17795_9693" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "14", + "height": "14", + "fill": "white" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "Tag01" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag01.tsx b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag01.tsx new file mode 100644 index 0000000000..1af1ac6f22 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag01.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './Tag01.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'Tag01' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag03.json b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag03.json new file mode 100644 index 0000000000..ef0753b8d3 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag03.json @@ -0,0 +1,39 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "16", + "height": "16", + "viewBox": "0 0 16 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "tag-03" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Icon", + "d": "M14 7.3335L8.93726 2.27075C8.59135 1.92485 8.4184 1.7519 8.21657 1.62822C8.03762 1.51856 7.84254 1.43775 7.63846 1.38876C7.40829 1.3335 7.16369 1.3335 6.67452 1.3335L4 1.3335M2 5.80016L2 7.11651C2 7.44263 2 7.60569 2.03684 7.75914C2.0695 7.89519 2.12337 8.02525 2.19648 8.14454C2.27894 8.2791 2.39424 8.3944 2.62484 8.625L7.82484 13.825C8.35286 14.353 8.61687 14.617 8.92131 14.716C9.1891 14.803 9.47757 14.803 9.74536 14.716C10.0498 14.617 10.3138 14.353 10.8418 13.825L12.4915 12.1753C13.0195 11.6473 13.2835 11.3833 13.3825 11.0789C13.4695 10.8111 13.4695 10.5226 13.3825 10.2548C13.2835 9.95037 13.0195 9.68636 12.4915 9.15834L7.62484 4.29167C7.39424 4.06107 7.27894 3.94577 7.14438 3.86331C7.02508 3.7902 6.89502 3.73633 6.75898 3.70367C6.60553 3.66683 6.44247 3.66683 6.11634 3.66683H4.13333C3.3866 3.66683 3.01323 3.66683 2.72801 3.81215C2.47713 3.93999 2.27316 4.14396 2.14532 4.39484C2 4.68006 2 5.05343 2 5.80016Z", + "stroke": "currentColor", + "stroke-width": "1.5", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + } + ] + }, + "name": "Tag03" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag03.tsx b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag03.tsx new file mode 100644 index 0000000000..654ea5c9ee --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Tag03.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './Tag03.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'Tag03' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/financeAndECommerce/index.ts b/web/app/components/base/icons/src/vender/line/financeAndECommerce/index.ts index a9ee6418b5..65c6ef70f3 100644 --- a/web/app/components/base/icons/src/vender/line/financeAndECommerce/index.ts +++ b/web/app/components/base/icons/src/vender/line/financeAndECommerce/index.ts @@ -1,3 +1,5 @@ export { default as CoinsStacked01 } from './CoinsStacked01' export { default as GoldCoin } from './GoldCoin' export { default as ReceiptList } from './ReceiptList' +export { default as Tag01 } from './Tag01' +export { default as Tag03 } from './Tag03' diff --git a/web/app/components/base/popover/index.tsx b/web/app/components/base/popover/index.tsx index 26d17c4405..92c7c34e36 100644 --- a/web/app/components/base/popover/index.tsx +++ b/web/app/components/base/popover/index.tsx @@ -13,7 +13,7 @@ type IPopover = { htmlContent: React.ReactElement popupClassName?: string trigger?: 'click' | 'hover' - position?: 'bottom' | 'br' + position?: 'bottom' | 'br' | 'bl' btnElement?: string | React.ReactNode btnClassName?: string | ((open: boolean) => string) manualClose?: boolean @@ -71,7 +71,13 @@ export default function CustomPopover({ + + + + +type Props = { + datasetId: string +} +type IIndexState = { + value: string +} +type ActionType = 'retry' | 'success' | 'error' + +type IAction = { + type: ActionType +} +const indexStateReducer = (state: IIndexState, action: IAction) => { + const actionMap = { + retry: 'retry', + success: 'success', + error: 'error', + } + + return { + ...state, + value: actionMap[action.type] || state.value, + } +} + +const RetryButton: FC = ({ datasetId }) => { + const { t } = useTranslation() + const [indexState, dispatch] = useReducer(indexStateReducer, { value: 'success' }) + const { data: errorDocs } = useSWR({ datasetId }, getErrorDocs) + + const onRetryErrorDocs = async () => { + dispatch({ type: 'retry' }) + const document_ids = errorDocs?.data.map((doc: IndexingStatusResponse) => doc.id) || [] + const res = await retryErrorDocs({ datasetId, document_ids }) + if (res.result === 'success') + dispatch({ type: 'success' }) + else + dispatch({ type: 'error' }) + } + + useEffect(() => { + if (errorDocs?.total === 0) + dispatch({ type: 'success' }) + else + dispatch({ type: 'error' }) + }, [errorDocs?.total]) + + if (indexState.value === 'success') + return null + + return ( +
+ + + {errorDocs?.total} {t('dataset.docsFailedNotice')} + + + + {t('dataset.retry')} + +
+ ) +} +export default RetryButton diff --git a/web/app/components/base/retry-button/style.module.css b/web/app/components/base/retry-button/style.module.css new file mode 100644 index 0000000000..99a0947576 --- /dev/null +++ b/web/app/components/base/retry-button/style.module.css @@ -0,0 +1,4 @@ +.retryBtn { + @apply inline-flex justify-center items-center content-center h-9 leading-5 rounded-lg px-4 py-2 text-base; + @apply border-solid border border-gray-200 text-gray-500 hover:bg-white hover:shadow-sm hover:border-gray-300; +} diff --git a/web/app/components/base/search-input/index.tsx b/web/app/components/base/search-input/index.tsx new file mode 100644 index 0000000000..b7a6e596cd --- /dev/null +++ b/web/app/components/base/search-input/index.tsx @@ -0,0 +1,66 @@ +import type { FC } from 'react' +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import cn from 'classnames' +import { SearchLg } from '@/app/components/base/icons/src/vender/line/general' +import { XCircle } from '@/app/components/base/icons/src/vender/solid/general' + +type SearchInputProps = { + placeholder?: string + className?: string + value: string + onChange: (v: string) => void + white?: boolean +} +const SearchInput: FC = ({ + placeholder, + className, + value, + onChange, + white, +}) => { + const { t } = useTranslation() + const [focus, setFocus] = useState(false) + + return ( +
+
+
+ { + onChange(e.target.value) + }} + onFocus={() => setFocus(true)} + onBlur={() => setFocus(false)} + autoComplete="off" + /> + {value && ( +
onChange('')} + > + +
+ )} +
+ ) +} + +export default SearchInput diff --git a/web/app/components/base/tag-management/constant.ts b/web/app/components/base/tag-management/constant.ts new file mode 100644 index 0000000000..3c60041383 --- /dev/null +++ b/web/app/components/base/tag-management/constant.ts @@ -0,0 +1,6 @@ +export type Tag = { + id: string + name: string + type: string + binding_count: number +} diff --git a/web/app/components/base/tag-management/filter.tsx b/web/app/components/base/tag-management/filter.tsx new file mode 100644 index 0000000000..d1c01bdbc1 --- /dev/null +++ b/web/app/components/base/tag-management/filter.tsx @@ -0,0 +1,142 @@ +import type { FC } from 'react' +import { useMemo, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { useDebounceFn, useMount } from 'ahooks' +import cn from 'classnames' +import { useStore as useTagStore } from './store' +import { + PortalToFollowElem, + PortalToFollowElemContent, + PortalToFollowElemTrigger, +} from '@/app/components/base/portal-to-follow-elem' +import SearchInput from '@/app/components/base/search-input' +import { ChevronDown } from '@/app/components/base/icons/src/vender/line/arrows' +import { Tag01, Tag03 } from '@/app/components/base/icons/src/vender/line/financeAndECommerce' +import { Check } from '@/app/components/base/icons/src/vender/line/general' +import { XCircle } from '@/app/components/base/icons/src/vender/solid/general' +import type { Tag } from '@/app/components/base/tag-management/constant' + +import { fetchTagList } from '@/service/tag' + +type TagFilterProps = { + type: 'knowledge' | 'app' + value: string[] + onChange: (v: string[]) => void +} +const TagFilter: FC = ({ + type, + value, + onChange, +}) => { + const { t } = useTranslation() + const [open, setOpen] = useState(false) + + const tagList = useTagStore(s => s.tagList) + const setTagList = useTagStore(s => s.setTagList) + + const [keywords, setKeywords] = useState('') + const [searchKeywords, setSearchKeywords] = useState('') + const { run: handleSearch } = useDebounceFn(() => { + setSearchKeywords(keywords) + }, { wait: 500 }) + const handleKeywordsChange = (value: string) => { + setKeywords(value) + handleSearch() + } + + const filteredTagList = useMemo(() => { + return tagList.filter(tag => tag.type === type && tag.name.includes(searchKeywords)) + }, [type, tagList, searchKeywords]) + + const currentTag = useMemo(() => { + return tagList.find(tag => tag.id === value[0]) + }, [value, tagList]) + + const selectTag = (tag: Tag) => { + if (value.includes(tag.id)) + onChange(value.filter(v => v !== tag.id)) + else + onChange([...value, tag.id]) + } + + useMount(() => { + fetchTagList(type).then((res) => { + setTagList(res) + }) + }) + + return ( + +
+ setOpen(v => !v)} + className='block' + > +
+
+ +
+
+ {!value.length && t('common.tag.placeholder')} + {!!value.length && currentTag?.name} +
+ {value.length > 1 && ( +
{`+${value.length - 1}`}
+ )} + {!value.length && ( +
+ +
+ )} + {!!value.length && ( +
{ + e.stopPropagation() + onChange([]) + }}> + +
+ )} +
+ + +
+
+ +
+
+ {filteredTagList.map(tag => ( +
selectTag(tag)} + > +
{tag.name}
+ {value.includes(tag.id) && } +
+ ))} + {!filteredTagList.length && ( +
+ +
{t('common.tag.noTag')}
+
+ )} +
+
+ +
+ + + ) +} + +export default TagFilter diff --git a/web/app/components/base/tag-management/index.tsx b/web/app/components/base/tag-management/index.tsx new file mode 100644 index 0000000000..5509a3e219 --- /dev/null +++ b/web/app/components/base/tag-management/index.tsx @@ -0,0 +1,93 @@ +'use client' + +import { useEffect, useState } from 'react' +import { useContext } from 'use-context-selector' +import { useTranslation } from 'react-i18next' +import { useStore as useTagStore } from './store' +import TagItemEditor from './tag-item-editor' +import Modal from '@/app/components/base/modal' +import { ToastContext } from '@/app/components/base/toast' +import { XClose } from '@/app/components/base/icons/src/vender/line/general' +import { + createTag, + fetchTagList, +} from '@/service/tag' + +type TagManagementModalProps = { + type: 'knowledge' | 'app' + show: boolean +} + +const TagManagementModal = ({ show, type }: TagManagementModalProps) => { + const { t } = useTranslation() + const { notify } = useContext(ToastContext) + const tagList = useTagStore(s => s.tagList) + const setTagList = useTagStore(s => s.setTagList) + const setShowTagManagementModal = useTagStore(s => s.setShowTagManagementModal) + + const getTagList = async (type: 'knowledge' | 'app') => { + const res = await fetchTagList(type) + setTagList(res) + } + + const [pending, setPending] = useState(false) + const [name, setName] = useState('') + const createNewTag = async () => { + if (!name) + return + if (pending) + return + try { + setPending(true) + const newTag = await createTag(name, type) + notify({ type: 'success', message: t('common.tag.created') }) + setTagList([ + newTag, + ...tagList, + ]) + setName('') + setPending(false) + } + catch (e: any) { + notify({ type: 'error', message: t('common.tag.failed') }) + setPending(false) + } + } + + useEffect(() => { + getTagList(type) + }, [type]) + + return ( + setShowTagManagementModal(false)} + > +
{t('common.tag.manageTags')}
+
setShowTagManagementModal(false)}> + +
+
+ setName(e.target.value)} + onKeyDown={e => e.key === 'Enter' && createNewTag()} + onBlur={createNewTag} + /> + {tagList.map(tag => ( + + ))} +
+ + ) +} + +export default TagManagementModal diff --git a/web/app/components/base/tag-management/selector.tsx b/web/app/components/base/tag-management/selector.tsx new file mode 100644 index 0000000000..9de25a34f0 --- /dev/null +++ b/web/app/components/base/tag-management/selector.tsx @@ -0,0 +1,272 @@ +import type { FC } from 'react' +import { useMemo, useState } from 'react' +import { useContext } from 'use-context-selector' +import { useTranslation } from 'react-i18next' +import { useUnmount } from 'ahooks' +import cn from 'classnames' +import { useStore as useTagStore } from './store' +import type { HtmlContentProps } from '@/app/components/base/popover' +import CustomPopover from '@/app/components/base/popover' +import Divider from '@/app/components/base/divider' +import SearchInput from '@/app/components/base/search-input' +import { Tag01, Tag03 } from '@/app/components/base/icons/src/vender/line/financeAndECommerce' +import { Plus } from '@/app/components/base/icons/src/vender/line/general' +import type { Tag } from '@/app/components/base/tag-management/constant' +import Checkbox from '@/app/components/base/checkbox' +import { bindTag, createTag, fetchTagList, unBindTag } from '@/service/tag' +import { ToastContext } from '@/app/components/base/toast' + +type TagSelectorProps = { + targetID: string + isPopover?: boolean + position?: 'bl' | 'br' + type: 'knowledge' | 'app' + value: string[] + selectedTags: Tag[] + onCacheUpdate: (tags: Tag[]) => void + onChange?: () => void +} + +type PanelProps = { + onCreate: () => void +} & HtmlContentProps & TagSelectorProps + +const Panel = (props: PanelProps) => { + const { t } = useTranslation() + const { notify } = useContext(ToastContext) + const { targetID, type, value, selectedTags, onCacheUpdate, onChange, onCreate } = props + const tagList = useTagStore(s => s.tagList) + const setTagList = useTagStore(s => s.setTagList) + const setShowTagManagementModal = useTagStore(s => s.setShowTagManagementModal) + const [selectedTagIDs, setSelectedTagIDs] = useState(value) + const [keywords, setKeywords] = useState('') + const handleKeywordsChange = (value: string) => { + setKeywords(value) + } + + const notExisted = useMemo(() => { + return tagList.every(tag => tag.type === type && tag.name !== keywords) + }, [type, tagList, keywords]) + const filteredSelectedTagList = useMemo(() => { + return selectedTags.filter(tag => tag.name.includes(keywords)) + }, [keywords, selectedTags]) + const filteredTagList = useMemo(() => { + return tagList.filter(tag => tag.type === type && !value.includes(tag.id) && tag.name.includes(keywords)) + }, [type, tagList, value, keywords]) + + const [creating, setCreating] = useState(false) + const createNewTag = async () => { + if (!keywords) + return + if (creating) + return + try { + setCreating(true) + const newTag = await createTag(keywords, type) + notify({ type: 'success', message: t('common.tag.created') }) + setTagList([ + ...tagList, + newTag, + ]) + setCreating(false) + onCreate() + } + catch (e: any) { + notify({ type: 'error', message: t('common.tag.failed') }) + setCreating(false) + } + } + const bind = async (tagIDs: string[]) => { + try { + await bindTag(tagIDs, targetID, type) + notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + } + catch (e: any) { + notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') }) + } + } + const unbind = async (tagID: string) => { + try { + await unBindTag(tagID, targetID, type) + notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + } + catch (e: any) { + notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') }) + } + } + const selectTag = (tag: Tag) => { + if (selectedTagIDs.includes(tag.id)) + setSelectedTagIDs(selectedTagIDs.filter(v => v !== tag.id)) + else + setSelectedTagIDs([...selectedTagIDs, tag.id]) + } + + const valueNotChanged = useMemo(() => { + return value.length === selectedTagIDs.length && value.every(v => selectedTagIDs.includes(v)) && selectedTagIDs.every(v => value.includes(v)) + }, [value, selectedTagIDs]) + const handleValueChange = () => { + const addTagIDs = selectedTagIDs.filter(v => !value.includes(v)) + const removeTagIDs = value.filter(v => !selectedTagIDs.includes(v)) + const selectedTags = tagList.filter(tag => selectedTagIDs.includes(tag.id)) + onCacheUpdate(selectedTags) + Promise.all([ + ...(addTagIDs.length ? [bind(addTagIDs)] : []), + ...[removeTagIDs.length ? removeTagIDs.map(tagID => unbind(tagID)) : []], + ]).finally(() => { + if (onChange) + onChange() + }) + } + useUnmount(() => { + if (valueNotChanged) + return + handleValueChange() + }) + + const onMouseLeave = async () => { + props.onClose?.() + } + return ( +
+
+ +
+ {keywords && notExisted && ( +
+
+ +
+ {`${t('common.tag.create')} `} + {`"${keywords}"`} +
+
+
+ )} + {keywords && notExisted && filteredTagList.length > 0 && ( + + )} + {(filteredTagList.length > 0 || filteredSelectedTagList.length > 0) && ( +
+ {filteredSelectedTagList.map(tag => ( +
selectTag(tag)} + > + {}} + /> +
{tag.name}
+
+ ))} + {filteredTagList.map(tag => ( +
selectTag(tag)} + > + {}} + /> +
{tag.name}
+
+ ))} +
+ )} + {!keywords && !filteredTagList.length && !filteredSelectedTagList.length && ( +
+
+ +
{t('common.tag.noTag')}
+
+
+ )} + +
+
setShowTagManagementModal(true)}> + +
+ {t('common.tag.manageTags')} +
+
+
+
+ ) +} + +const TagSelector: FC = ({ + targetID, + isPopover = true, + position, + type, + value, + selectedTags, + onCacheUpdate, + onChange, +}) => { + const { t } = useTranslation() + + const setTagList = useTagStore(s => s.setTagList) + + const getTagList = async () => { + const res = await fetchTagList(type) + setTagList(res) + } + + const triggerContent = useMemo(() => { + if (selectedTags?.length) + return selectedTags.map(tag => tag.name).join(', ') + return '' + }, [selectedTags]) + + const Trigger = () => { + return ( +
+ +
+ {!triggerContent ? t('common.tag.addTag') : triggerContent} +
+ {t('common.tag.editTag')} +
+ ) + } + return ( + <> + {isPopover && ( + + } + position={position} + trigger="click" + btnElement={} + btnClassName={open => + cn( + open ? '!bg-gray-100 !text-gray-700' : '!bg-transparent', + '!w-full !p-0 !border-0 !text-gray-500 hover:!bg-gray-100 hover:!text-gray-700', + ) + } + popupClassName='!w-full !ring-0' + className={'!w-full h-fit !z-20'} + /> + )} + + + ) +} + +export default TagSelector diff --git a/web/app/components/base/tag-management/store.ts b/web/app/components/base/tag-management/store.ts new file mode 100644 index 0000000000..cb92ae9764 --- /dev/null +++ b/web/app/components/base/tag-management/store.ts @@ -0,0 +1,19 @@ +import { create } from 'zustand' +import type { Tag } from './constant' + +type State = { + tagList: Tag[] + showTagManagementModal: boolean +} + +type Action = { + setTagList: (tagList?: Tag[]) => void + setShowTagManagementModal: (showTagManagementModal: boolean) => void +} + +export const useStore = create(set => ({ + tagList: [], + setTagList: tagList => set(() => ({ tagList })), + showTagManagementModal: false, + setShowTagManagementModal: showTagManagementModal => set(() => ({ showTagManagementModal })), +})) diff --git a/web/app/components/base/tag-management/style.module.css b/web/app/components/base/tag-management/style.module.css new file mode 100644 index 0000000000..14367ec575 --- /dev/null +++ b/web/app/components/base/tag-management/style.module.css @@ -0,0 +1,3 @@ +.bg { + background: linear-gradient(180deg, rgba(247, 144, 9, 0.05) 0%, rgba(247, 144, 9, 0.00) 24.41%), #F9FAFB; +} diff --git a/web/app/components/base/tag-management/tag-item-editor.tsx b/web/app/components/base/tag-management/tag-item-editor.tsx new file mode 100644 index 0000000000..d0a6989fd4 --- /dev/null +++ b/web/app/components/base/tag-management/tag-item-editor.tsx @@ -0,0 +1,147 @@ +import type { FC } from 'react' +import { useState } from 'react' +import cn from 'classnames' +import { useDebounceFn } from 'ahooks' +import { useContext } from 'use-context-selector' +import { useTranslation } from 'react-i18next' +import { useStore as useTagStore } from './store' +import TagRemoveModal from './tag-remove-modal' +import { Edit03, Trash03 } from '@/app/components/base/icons/src/vender/line/general' +import type { Tag } from '@/app/components/base/tag-management/constant' +import { ToastContext } from '@/app/components/base/toast' +import { + deleteTag, + updateTag, +} from '@/service/tag' + +type TagItemEditorProps = { + tag: Tag +} +const TagItemEditor: FC = ({ + tag, +}) => { + const { t } = useTranslation() + const { notify } = useContext(ToastContext) + const tagList = useTagStore(s => s.tagList) + const setTagList = useTagStore(s => s.setTagList) + + const [isEditing, setIsEditing] = useState(false) + const [name, setName] = useState(tag.name) + const editTag = async (tagID: string, name: string) => { + if (name === tag.name) { + setIsEditing(false) + return + } + if (!name) { + notify({ type: 'error', message: 'tag name is empty' }) + setName(tag.name) + setIsEditing(false) + return + } + try { + const newList = tagList.map((tag) => { + if (tag.id === tagID) { + return { + ...tag, + name, + } + } + return tag + }) + setTagList([ + ...newList, + ]) + setIsEditing(false) + await updateTag(tagID, name) + notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + setName(name) + } + catch (e: any) { + notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') }) + setName(tag.name) + const recoverList = tagList.map((tag) => { + if (tag.id === tagID) { + return { + ...tag, + name: tag.name, + } + } + return tag + }) + setTagList([ + ...recoverList, + ]) + setIsEditing(false) + } + } + const [showRemoveModal, setShowRemoveModal] = useState(false) + const [pending, setPending] = useState(false) + const removeTag = async (tagID: string) => { + if (pending) + return + try { + setPending(true) + await deleteTag(tagID) + notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + const newList = tagList.filter(tag => tag.id !== tagID) + setTagList([ + ...newList, + ]) + setPending(false) + } + catch (e: any) { + notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') }) + setPending(false) + } + } + const { run: handleRemove } = useDebounceFn(() => { + removeTag(tag.id) + }, { wait: 200 }) + + return ( + <> +
+ {!isEditing && ( + <> +
+ {tag.name} +
+
{tag.binding_count}
+
setIsEditing(true)}> + +
+
{ + if (tag.binding_count) + setShowRemoveModal(true) + else + handleRemove() + }}> + +
+ + )} + {isEditing && ( + setName(e.target.value)} + onKeyDown={e => e.key === 'Enter' && editTag(tag.id, name)} + onBlur={() => editTag(tag.id, name)} + /> + )} +
+ { + handleRemove() + setShowRemoveModal(false) + }} + onClose={() => setShowRemoveModal(false)} + /> + + ) +} + +export default TagItemEditor diff --git a/web/app/components/base/tag-management/tag-remove-modal.tsx b/web/app/components/base/tag-management/tag-remove-modal.tsx new file mode 100644 index 0000000000..13400a074b --- /dev/null +++ b/web/app/components/base/tag-management/tag-remove-modal.tsx @@ -0,0 +1,50 @@ +'use client' + +import { useTranslation } from 'react-i18next' +import cn from 'classnames' +import s from './style.module.css' +import Button from '@/app/components/base/button' +import Modal from '@/app/components/base/modal' +import { XClose } from '@/app/components/base/icons/src/vender/line/general' +import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback' +import type { Tag } from '@/app/components/base/tag-management/constant' + +type TagRemoveModalProps = { + show: boolean + tag: Tag + onConfirm: () => void + onClose: () => void +} + +const TagRemoveModal = ({ show, tag, onConfirm, onClose }: TagRemoveModalProps) => { + const { t } = useTranslation() + + return ( + {}} + > +
+ +
+
+ +
+
+ {`${t('common.tag.delete')} `} + {`"${tag.name}"`} +
+
+ {t('common.tag.deleteTip')} +
+
+ + +
+ + ) +} + +export default TagRemoveModal diff --git a/web/app/components/datasets/documents/index.tsx b/web/app/components/datasets/documents/index.tsx index 13f8fd8f9b..cf62d88528 100644 --- a/web/app/components/datasets/documents/index.tsx +++ b/web/app/components/datasets/documents/index.tsx @@ -5,7 +5,6 @@ import useSWR from 'swr' import { useTranslation } from 'react-i18next' import { useRouter } from 'next/navigation' import { debounce, groupBy, omit } from 'lodash-es' -// import Link from 'next/link' import { PlusIcon } from '@heroicons/react/24/solid' import List from './list' import s from './style.module.css' @@ -20,7 +19,7 @@ import { NotionPageSelectorModal } from '@/app/components/base/notion-page-selec import type { NotionPage } from '@/models/common' import type { CreateDocumentReq } from '@/models/datasets' import { DataSourceType } from '@/models/datasets' - +import RetryButton from '@/app/components/base/retry-button' // Custom page count is not currently supported. const limit = 15 @@ -198,7 +197,7 @@ const Documents: FC = ({ datasetId }) => {

{t('datasetDocuments.list.desc')}

-
+
= ({ datasetId }) => { onChange={debounce(setSearchValue, 500)} value={searchValue} /> - {embeddingAvailable && ( - - )} +
+ + {embeddingAvailable && ( + + )} +
{isLoading ? diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index d83e6a4bea..046032431e 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -332,7 +332,7 @@ const DocumentList: FC = ({ embeddingAvailable, documents = # {t('datasetDocuments.list.table.header.fileName')} {t('datasetDocuments.list.table.header.words')} - {t('datasetDocuments.list.table.header.hitCount')} + {t('datasetDocuments.list.table.header.hitCount')}
{t('datasetDocuments.list.table.header.uploadTime')} diff --git a/web/app/components/datasets/rename-modal/index.tsx b/web/app/components/datasets/rename-modal/index.tsx new file mode 100644 index 0000000000..4245b3112f --- /dev/null +++ b/web/app/components/datasets/rename-modal/index.tsx @@ -0,0 +1,106 @@ +'use client' + +import type { MouseEventHandler } from 'react' +import cn from 'classnames' +import { useState } from 'react' +import { BookOpenIcon } from '@heroicons/react/24/outline' +import { useContext } from 'use-context-selector' +import { useTranslation } from 'react-i18next' +import Button from '@/app/components/base/button' +import Modal from '@/app/components/base/modal' +import { ToastContext } from '@/app/components/base/toast' +import { XClose } from '@/app/components/base/icons/src/vender/line/general' +import type { DataSet } from '@/models/datasets' +import { updateDatasetSetting } from '@/service/datasets' + +type RenameDatasetModalProps = { + show: boolean + dataset: DataSet + onSuccess?: () => void + onClose: () => void +} + +const RenameDatasetModal = ({ show, dataset, onSuccess, onClose }: RenameDatasetModalProps) => { + const { t } = useTranslation() + const { notify } = useContext(ToastContext) + const [loading, setLoading] = useState(false) + const [name, setName] = useState(dataset.name) + const [description, setDescription] = useState(dataset.description) + + const onConfirm: MouseEventHandler = async () => { + if (!name.trim()) { + notify({ type: 'error', message: t('datasetSettings.form.nameError') }) + return + } + try { + setLoading(true) + await updateDatasetSetting({ + datasetId: dataset.id, + body: { + name, + description, + }, + }) + notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + if (onSuccess) + onSuccess() + onClose() + } + catch (e) { + notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') }) + } + finally { + setLoading(false) + } + } + + return ( + {}} + > +
{t('datasetSettings.title')}
+
+ +
+
+
+
+ {t('datasetSettings.form.name')} +
+ setName(e.target.value)} + className='block px-3 w-full h-9 bg-gray-100 rounded-lg text-sm text-gray-900 outline-none appearance-none' + placeholder={t('datasetSettings.form.namePlaceholder') || ''} + /> +
+
+
+ {t('datasetSettings.form.desc')} +
+
+