enhancement: introduce Ruff for Python linter for reordering and removing unused imports with automated pre-commit and sytle check (#2366)

.github/workflows/style.yml

@@ -10,9 +10,33 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  python-style:
+    name: Python Style
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Python dependencies
+        run: pip install ruff
+
+      - name: Ruff check
+        run: ruff check ./api
+
+      - name: Lint hints
+        if: failure()
+        run: echo "Please run 'dev/reformat' to fix the fixable linting errors."
+
   test:
     name: ESLint and SuperLinter
     runs-on: ubuntu-latest
+    needs: python-style
 
     steps:
       - name: Checkout code

api/app.py

@@ -19,18 +19,28 @@ import threading
 import time
 import warnings
 
-from commands import register_commands
-from config import CloudEditionConfig, Config
-from events import event_handlers
-from extensions import (ext_celery, ext_code_based_extension, ext_database, ext_hosting_provider, ext_login, ext_mail,
-                        ext_migrate, ext_redis, ext_sentry, ext_storage)
-from extensions.ext_database import db
-from extensions.ext_login import login_manager
 from flask import Flask, Response, request
 from flask_cors import CORS
+
+from commands import register_commands
+from config import CloudEditionConfig, Config
+from extensions import (
+    ext_celery,
+    ext_code_based_extension,
+    ext_database,
+    ext_hosting_provider,
+    ext_login,
+    ext_mail,
+    ext_migrate,
+    ext_redis,
+    ext_sentry,
+    ext_storage,
+)
+from extensions.ext_database import db
+from extensions.ext_login import login_manager
 from libs.passport import PassportService
+
 # DO NOT REMOVE BELOW
-from models import account, dataset, model, source, task, tool, tools, web
 from services.account_service import AccountService
 
 # DO NOT REMOVE ABOVE

api/commands.py

@@ -3,11 +3,13 @@ import json
 import secrets
 
 import click
+from flask import current_app
+from werkzeug.exceptions import NotFound
+
 from core.embedding.cached_embedding import CacheEmbedding
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from extensions.ext_database import db
-from flask import current_app
 from libs.helper import email as email_validate
 from libs.password import hash_password, password_pattern, valid_password
 from libs.rsa import generate_key_pair
@@ -15,7 +17,6 @@ from models.account import Tenant
 from models.dataset import Dataset
 from models.model import Account
 from models.provider import Provider, ProviderModel
-from werkzeug.exceptions import NotFound
 
 
 @click.command('reset-password', help='Reset the account password.')

api/constants/model_template.py

@@ -1,7 +1,5 @@
 import json
 
-from models.model import App, AppModelConfig
-
 model_templates = {
     # completion default mode
     'completion_default': {

api/controllers/console/admin.py

@@ -1,14 +1,15 @@
 import os
 from functools import wraps
 
+from flask import request
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import NotFound, Unauthorized
+
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.wraps import only_edition_cloud
 from extensions.ext_database import db
-from flask import request
-from flask_restful import Resource, reqparse
 from models.model import App, InstalledApp, RecommendedApp
-from werkzeug.exceptions import NotFound, Unauthorized
 
 
 def admin_required(view):

api/controllers/console/apikey.py

@@ -1,12 +1,13 @@
 import flask_restful
-from extensions.ext_database import db
 from flask_login import current_user
 from flask_restful import Resource, fields, marshal_with
+from werkzeug.exceptions import Forbidden
+
+from extensions.ext_database import db
 from libs.helper import TimestampField
 from libs.login import login_required
 from models.dataset import Dataset
 from models.model import ApiToken, App
-from werkzeug.exceptions import Forbidden
 
 from . import api
 from .setup import setup_required

api/controllers/console/app/advanced_prompt_template.py

@@ -1,7 +1,8 @@
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.advanced_prompt_template_service import AdvancedPromptTemplateService
 

api/controllers/console/app/annotation.py

@@ -1,17 +1,20 @@
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden
+
 from controllers.console import api
 from controllers.console.app.error import NoFileUploadedError
 from controllers.console.datasets.error import TooManyFilesError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_redis import redis_client
-from fields.annotation_fields import (annotation_fields, annotation_hit_history_fields,
-                                      annotation_hit_history_list_fields, annotation_list_fields)
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse
+from fields.annotation_fields import (
+    annotation_fields,
+    annotation_hit_history_fields,
+)
 from libs.login import login_required
 from services.annotation_service import AppAnnotationService
-from werkzeug.exceptions import Forbidden
 
 
 class AnnotationReplyActionApi(Resource):

api/controllers/console/app/app.py

@@ -3,6 +3,10 @@ import json
 import logging
 from datetime import datetime
 
+from flask_login import current_user
+from flask_restful import Resource, abort, inputs, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden
+
 from constants.languages import demo_model_templates, languages
 from constants.model_template import model_templates
 from controllers.console import api
@@ -15,16 +19,15 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from events.app_event import app_was_created, app_was_deleted
 from extensions.ext_database import db
-from fields.app_fields import (app_detail_fields, app_detail_fields_with_site, app_pagination_fields,
-                               template_list_fields)
-from flask import current_app
-from flask_login import current_user
-from flask_restful import Resource, abort, inputs, marshal_with, reqparse
+from fields.app_fields import (
+    app_detail_fields,
+    app_detail_fields_with_site,
+    app_pagination_fields,
+    template_list_fields,
+)
 from libs.login import login_required
 from models.model import App, AppModelConfig, Site
-from models.tools import ApiToolProvider
 from services.app_model_config_service import AppModelConfigService
-from werkzeug.exceptions import Forbidden
 
 
 def _get_app(app_id, tenant_id):

api/controllers/console/app/audio.py

@@ -1,24 +1,36 @@
 # -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from flask_restful import Resource
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.console import api
 from controllers.console.app import _get_app
-from controllers.console.app.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                           NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                           ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
-from flask_restful import Resource
 from libs.login import login_required
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class ChatMessageAudioApi(Resource):

api/controllers/console/app/completion.py

@@ -4,24 +4,30 @@ import logging
 from typing import Generator, Union
 
 import flask_login
+from flask import Response, stream_with_context
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.console import api
 from controllers.console.app import _get_app
-from controllers.console.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.application_queue_manager import ApplicationQueueManager
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import Response, stream_with_context
-from flask_restful import Resource, reqparse
 from libs.helper import uuid_value
 from libs.login import login_required
 from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 # define completion message api for user

api/controllers/console/app/conversation.py

@@ -1,22 +1,27 @@
 from datetime import datetime
 
 import pytz
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from sqlalchemy import func, or_
+from sqlalchemy.orm import joinedload
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from fields.conversation_fields import (conversation_detail_fields, conversation_message_detail_fields,
-                                        conversation_pagination_fields, conversation_with_summary_pagination_fields)
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
-from flask_restful.inputs import int_range
+from fields.conversation_fields import (
+    conversation_detail_fields,
+    conversation_message_detail_fields,
+    conversation_pagination_fields,
+    conversation_with_summary_pagination_fields,
+)
 from libs.helper import datetime_string
 from libs.login import login_required
 from models.model import Conversation, Message, MessageAnnotation
-from sqlalchemy import func, or_
-from sqlalchemy.orm import joinedload
-from werkzeug.exceptions import NotFound
 
 
 class CompletionConversationApi(Resource):

api/controllers/console/app/generator.py

@@ -1,13 +1,18 @@
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
-from controllers.console.app.error import (CompletionRequestError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.generator.llm_generator import LLMGenerator
 from core.model_runtime.errors.invoke import InvokeError
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 
 

api/controllers/console/app/message.py

@@ -2,11 +2,21 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_login import current_user
+from flask_restful import Resource, fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
 from controllers.console import api
 from controllers.console.app import _get_app
-from controllers.console.app.error import (AppMoreLikeThisDisabledError, CompletionRequestError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    AppMoreLikeThisDisabledError,
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from core.entities.application_entities import InvokeFrom
@@ -14,10 +24,6 @@ from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotIni
 from core.model_runtime.errors.invoke import InvokeError
 from extensions.ext_database import db
 from fields.conversation_fields import annotation_fields, message_detail_fields
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import uuid_value
 from libs.infinite_scroll_pagination import InfiniteScrollPagination
 from libs.login import login_required
@@ -28,7 +34,6 @@ from services.errors.app import MoreLikeThisDisabledError
 from services.errors.conversation import ConversationNotExistsError
 from services.errors.message import MessageNotExistsError
 from services.message_service import MessageService
-from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 
 class ChatMessageListApi(Resource):

api/controllers/console/app/model_config.py

@@ -1,14 +1,15 @@
 # -*- coding:utf-8 -*-
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource
+
 from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from events.app_event import app_model_config_was_updated
 from extensions.ext_database import db
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource
 from libs.login import login_required
 from models.model import AppModelConfig
 from services.app_model_config_service import AppModelConfigService

api/controllers/console/app/site.py

@@ -1,4 +1,8 @@
 # -*- coding:utf-8 -*-
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.app import _get_app
@@ -6,11 +10,8 @@ from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from fields.app_fields import app_site_fields
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
 from libs.login import login_required
 from models.model import Site
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 def parse_app_site_args():

api/controllers/console/app/statistic.py

@@ -3,14 +3,15 @@ from datetime import datetime
 from decimal import Decimal
 
 import pytz
+from flask import jsonify
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from flask import jsonify
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.helper import datetime_string
 from libs.login import login_required
 

api/controllers/console/auth/activate.py

@@ -2,14 +2,15 @@ import base64
 import secrets
 from datetime import datetime
 
+from flask_restful import Resource, reqparse
+
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.error import AlreadyActivateError
 from extensions.ext_database import db
-from flask_restful import Resource, reqparse
 from libs.helper import email, str_len, timezone
 from libs.password import hash_password, valid_password
-from models.account import AccountStatus, Tenant
+from models.account import AccountStatus
 from services.account_service import RegisterService
 
 

api/controllers/console/auth/data_source_oauth.py

@@ -1,13 +1,14 @@
 import logging
 
 import requests
-from controllers.console import api
 from flask import current_app, redirect, request
 from flask_login import current_user
 from flask_restful import Resource
+from werkzeug.exceptions import Forbidden
+
+from controllers.console import api
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
-from werkzeug.exceptions import Forbidden
 
 from ..setup import setup_required
 from ..wraps import account_initialization_required

api/controllers/console/auth/login.py

@@ -1,11 +1,11 @@
 # -*- coding:utf-8 -*-
-import flask
 import flask_login
+from flask import current_app, request
+from flask_restful import Resource, reqparse
+
 import services
 from controllers.console import api
 from controllers.console.setup import setup_required
-from flask import current_app, request
-from flask_restful import Resource, reqparse
 from libs.helper import email
 from libs.password import valid_password
 from services.account_service import AccountService

api/controllers/console/auth/oauth.py

@@ -3,10 +3,11 @@ from datetime import datetime
 from typing import Optional
 
 import requests
-from constants.languages import languages
-from extensions.ext_database import db
 from flask import current_app, redirect, request
 from flask_restful import Resource
+
+from constants.languages import languages
+from extensions.ext_database import db
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models.account import Account, AccountStatus
 from services.account_service import AccountService, RegisterService

api/controllers/console/billing/billing.py

@@ -1,8 +1,9 @@
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, only_edition_cloud
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.billing_service import BillingService
 

api/controllers/console/datasets/data_source.py

@@ -1,6 +1,11 @@
 import datetime
 import json
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
@@ -8,15 +13,11 @@ from core.data_loader.loader.notion import NotionLoader
 from core.indexing_runner import IndexingRunner
 from extensions.ext_database import db
 from fields.data_source_fields import integrate_list_fields, integrate_notion_info_list_fields
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
 from libs.login import login_required
 from models.dataset import Document
 from models.source import DataSourceBinding
 from services.dataset_service import DatasetService, DocumentService
 from tasks.document_indexing_sync_task import document_indexing_sync_task
-from werkzeug.exceptions import NotFound
 
 
 class DataSourceApi(Resource):

api/controllers/console/datasets/datasets.py

@@ -1,5 +1,10 @@
 # -*- coding:utf-8 -*-
 import flask_restful
+from flask import current_app, request
+from flask_login import current_user
+from flask_restful import Resource, marshal, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
 import services
 from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
@@ -15,14 +20,10 @@ from extensions.ext_database import db
 from fields.app_fields import related_app_list
 from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
 from fields.document_fields import document_status_fields
-from flask import current_app, request
-from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse
 from libs.login import login_required
 from models.dataset import Dataset, Document, DocumentSegment
 from models.model import ApiToken, UploadFile
 from services.dataset_service import DatasetService, DocumentService
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 def _validate_name(name):

api/controllers/console/datasets/datasets_document.py

@@ -2,35 +2,52 @@
 from datetime import datetime
 from typing import List
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, fields, marshal, marshal_with, reqparse
+from sqlalchemy import asc, desc
+from werkzeug.exceptions import Forbidden, NotFound
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
-from controllers.console.datasets.error import (ArchivedDocumentImmutableError, DocumentAlreadyFinishedError,
-                                                DocumentIndexingError, InvalidActionError, InvalidMetadataError)
+from controllers.console.app.error import (
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.datasets.error import (
+    ArchivedDocumentImmutableError,
+    DocumentAlreadyFinishedError,
+    DocumentIndexingError,
+    InvalidActionError,
+    InvalidMetadataError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
-from core.errors.error import (LLMBadRequestError, ModelCurrentlyNotSupportError, ProviderTokenNotInitError,
-                               QuotaExceededError)
+from core.errors.error import (
+    LLMBadRequestError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.indexing_runner import IndexingRunner
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.invoke import InvokeAuthorizationError
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from fields.document_fields import (dataset_and_document_fields, document_fields, document_status_fields,
-                                    document_with_segments_fields)
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal, marshal_with, reqparse
+from fields.document_fields import (
+    dataset_and_document_fields,
+    document_fields,
+    document_status_fields,
+    document_with_segments_fields,
+)
 from libs.login import login_required
 from models.dataset import Dataset, DatasetProcessRule, Document, DocumentSegment
 from models.model import UploadFile
 from services.dataset_service import DatasetService, DocumentService
-from sqlalchemy import asc, desc
 from tasks.add_document_to_index_task import add_document_to_index_task
 from tasks.remove_document_from_index_task import remove_document_from_index_task
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 class DocumentResource(Resource):

api/controllers/console/datasets/datasets_segments.py

@@ -3,6 +3,11 @@ import uuid
 from datetime import datetime
 
 import pandas as pd
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
 import services
 from controllers.console import api
 from controllers.console.app.error import ProviderNotInitializeError
@@ -15,16 +20,12 @@ from core.model_runtime.entities.model_entities import ModelType
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from fields.segment_fields import segment_fields
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse
 from libs.login import login_required
 from models.dataset import DocumentSegment
 from services.dataset_service import DatasetService, DocumentService, SegmentService
 from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
 from tasks.disable_segment_from_index_task import disable_segment_from_index_task
 from tasks.enable_segment_to_index_task import enable_segment_to_index_task
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 class DatasetDocumentSegmentListApi(Resource):

api/controllers/console/datasets/file.py

@@ -1,13 +1,18 @@
-import services
-from controllers.console import api
-from controllers.console.datasets.error import (FileTooLargeError, NoFileUploadedError, TooManyFilesError,
-                                                UnsupportedFileTypeError)
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
-from fields.file_fields import file_fields, upload_config_fields
 from flask import current_app, request
 from flask_login import current_user
 from flask_restful import Resource, marshal_with
+
+import services
+from controllers.console import api
+from controllers.console.datasets.error import (
+    FileTooLargeError,
+    NoFileUploadedError,
+    TooManyFilesError,
+    UnsupportedFileTypeError,
+)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from fields.file_fields import file_fields, upload_config_fields
 from libs.login import login_required
 from services.file_service import ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS, FileService
 

api/controllers/console/datasets/hit_testing.py

@@ -1,22 +1,31 @@
 import logging
 
+from flask_login import current_user
+from flask_restful import Resource, marshal, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (CompletionRequestError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.datasets.error import DatasetNotInitializedError, HighQualityDatasetOnlyError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from core.errors.error import (LLMBadRequestError, ModelCurrentlyNotSupportError, ProviderTokenNotInitError,
-                               QuotaExceededError)
+from core.errors.error import (
+    LLMBadRequestError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.model_runtime.errors.invoke import InvokeError
 from fields.hit_testing_fields import hit_testing_record_fields
-from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse
 from libs.login import login_required
 from services.dataset_service import DatasetService
 from services.hit_testing_service import HitTestingService
-from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 
 class HitTestingApi(Resource):

api/controllers/console/explore/audio.py

@@ -1,21 +1,33 @@
 # -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                           NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                           ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
 from controllers.console.explore.wraps import InstalledAppResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
 from models.model import AppModelConfig
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class ChatAudioApi(InstalledAppResource):

api/controllers/console/explore/completion.py

@@ -4,11 +4,21 @@ import logging
 from datetime import datetime
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_login import current_user
+from flask_restful import reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.explore.error import NotChatAppError, NotCompletionAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from core.application_queue_manager import ApplicationQueueManager
@@ -16,12 +26,8 @@ from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from extensions.ext_database import db
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import reqparse
 from libs.helper import uuid_value
 from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 # define completion api for user

api/controllers/console/explore/conversation.py

@@ -1,16 +1,17 @@
 # -*- coding:utf-8 -*-
+from flask_login import current_user
+from flask_restful import marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.explore.error import NotChatAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
+from libs.helper import uuid_value
 from services.conversation_service import ConversationService
 from services.errors.conversation import ConversationNotExistsError, LastConversationNotExistsError
 from services.web_conversation_service import WebConversationService
-from werkzeug.exceptions import NotFound
 
 
 class ConversationListApi(InstalledAppResource):

api/controllers/console/explore/installed_app.py

@@ -1,18 +1,19 @@
 # -*- coding:utf-8 -*-
 from datetime import datetime
 
+from flask_login import current_user
+from flask_restful import Resource, inputs, marshal_with, reqparse
+from sqlalchemy import and_
+from werkzeug.exceptions import BadRequest, Forbidden, NotFound
+
 from controllers.console import api
 from controllers.console.explore.wraps import InstalledAppResource
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
 from fields.installed_app_fields import installed_app_list_fields
-from flask_login import current_user
-from flask_restful import Resource, inputs, marshal_with, reqparse
 from libs.login import login_required
 from models.model import App, InstalledApp, RecommendedApp
 from services.account_service import TenantService
-from sqlalchemy import and_
-from werkzeug.exceptions import BadRequest, Forbidden, NotFound
 
 
 class InstalledAppsListApi(Resource):

api/controllers/console/explore/message.py

@@ -3,29 +3,37 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_login import current_user
+from flask_restful import marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (AppMoreLikeThisDisabledError, CompletionRequestError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
-from controllers.console.explore.error import (AppSuggestedQuestionsAfterAnswerDisabledError, NotChatAppError,
-                                               NotCompletionAppError)
+from controllers.console.app.error import (
+    AppMoreLikeThisDisabledError,
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.explore.error import (
+    AppSuggestedQuestionsAfterAnswerDisabledError,
+    NotChatAppError,
+    NotCompletionAppError,
+)
 from controllers.console.explore.wraps import InstalledAppResource
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from fields.message_fields import message_infinite_scroll_pagination_fields
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
+from libs.helper import uuid_value
 from services.completion_service import CompletionService
 from services.errors.app import MoreLikeThisDisabledError
 from services.errors.conversation import ConversationNotExistsError
 from services.errors.message import MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError
 from services.message_service import MessageService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 class MessageListApi(InstalledAppResource):

api/controllers/console/explore/parameter.py

@@ -1,11 +1,12 @@
 # -*- coding:utf-8 -*-
 import json
 
+from flask import current_app
+from flask_restful import fields, marshal_with
+
 from controllers.console import api
 from controllers.console.explore.wraps import InstalledAppResource
 from extensions.ext_database import db
-from flask import current_app
-from flask_restful import fields, marshal_with
 from models.model import AppModelConfig, InstalledApp
 from models.tools import ApiToolProvider
 

api/controllers/console/explore/recommended_app.py

@@ -1,15 +1,16 @@
 # -*- coding:utf-8 -*-
+from flask_login import current_user
+from flask_restful import Resource, fields, marshal_with
+from sqlalchemy import and_
+
 from constants.languages import languages
 from controllers.console import api
 from controllers.console.app.error import AppNotFoundError
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with
 from libs.login import login_required
 from models.model import App, InstalledApp, RecommendedApp
 from services.account_service import TenantService
-from sqlalchemy import and_
 
 app_fields = {
     'id': fields.String,

api/controllers/console/explore/saved_message.py

@@ -1,14 +1,15 @@
+from flask_login import current_user
+from flask_restful import fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.explore.error import NotCompletionAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from fields.conversation_fields import message_file_fields
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import TimestampField, uuid_value
 from services.errors.message import MessageNotExistsError
 from services.saved_message_service import SavedMessageService
-from werkzeug.exceptions import NotFound
 
 feedback_fields = {
     'rating': fields.String

api/controllers/console/explore/wraps.py

@@ -1,12 +1,13 @@
 from functools import wraps
 
+from flask_login import current_user
+from flask_restful import Resource
+from werkzeug.exceptions import NotFound
+
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from flask_login import current_user
-from flask_restful import Resource
 from libs.login import login_required
 from models.model import InstalledApp
-from werkzeug.exceptions import NotFound
 
 
 def installed_app_required(view=None):

api/controllers/console/extension.py

@@ -1,9 +1,10 @@
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from fields.api_based_extension_fields import api_based_extension_fields
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
 from libs.login import login_required
 from models.api_based_extension import APIBasedExtension
 from services.api_based_extension_service import APIBasedExtensionService

api/controllers/console/feature.py

@@ -1,5 +1,6 @@
 from flask_login import current_user
 from flask_restful import Resource
+
 from services.feature_service import FeatureService
 
 from . import api

api/controllers/console/init_validate.py

@@ -2,6 +2,7 @@ import os
 
 from flask import current_app, session
 from flask_restful import Resource, reqparse
+
 from libs.helper import str_len
 from models.model import DifySetup
 from services.account_service import TenantService

api/controllers/console/setup.py

@@ -1,9 +1,10 @@
 # -*- coding:utf-8 -*-
 from functools import wraps
 
-from extensions.ext_database import db
 from flask import current_app, request
 from flask_restful import Resource, reqparse
+
+from extensions.ext_database import db
 from libs.helper import email, str_len
 from libs.password import valid_password
 from models.model import DifySetup

api/controllers/console/version.py

@@ -6,7 +6,6 @@ import logging
 import requests
 from flask import current_app
 from flask_restful import Resource, reqparse
-from werkzeug.exceptions import InternalServerError
 
 from . import api
 

api/controllers/console/workspace/account.py

@@ -2,16 +2,21 @@
 from datetime import datetime
 
 import pytz
-from constants.languages import supported_language
-from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.workspace.error import (AccountAlreadyInitedError, CurrentPasswordIncorrectError,
-                                                 InvalidInvitationCodeError, RepeatPasswordNotMatchError)
-from controllers.console.wraps import account_initialization_required
-from extensions.ext_database import db
 from flask import current_app, request
 from flask_login import current_user
 from flask_restful import Resource, fields, marshal_with, reqparse
+
+from constants.languages import supported_language
+from controllers.console import api
+from controllers.console.setup import setup_required
+from controllers.console.workspace.error import (
+    AccountAlreadyInitedError,
+    CurrentPasswordIncorrectError,
+    InvalidInvitationCodeError,
+    RepeatPasswordNotMatchError,
+)
+from controllers.console.wraps import account_initialization_required
+from extensions.ext_database import db
 from libs.helper import TimestampField, timezone
 from libs.login import login_required
 from models.account import AccountIntegrate, InvitationCode

api/controllers/console/workspace/members.py

@@ -1,12 +1,13 @@
 # -*- coding:utf-8 -*-
+from flask import current_app
+from flask_login import current_user
+from flask_restful import Resource, abort, fields, marshal_with, reqparse
+
 import services
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
-from flask import current_app
-from flask_login import current_user
-from flask_restful import Resource, abort, fields, marshal_with, reqparse
 from libs.helper import TimestampField
 from libs.login import login_required
 from models.account import Account

api/controllers/console/workspace/model_providers.py

@@ -1,18 +1,19 @@
 import io
 
+from flask import send_file
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import Forbidden
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder
-from flask import send_file
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.billing_service import BillingService
 from services.model_provider_service import ModelProviderService
-from werkzeug.exceptions import Forbidden
 
 
 class ModelProviderListApi(Resource):

api/controllers/console/workspace/models.py

@@ -1,16 +1,17 @@
 import logging
 
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import Forbidden
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.model_provider_service import ModelProviderService
-from werkzeug.exceptions import Forbidden
 
 
 class DefaultModelApi(Resource):

api/controllers/console/workspace/tool_providers.py

@@ -1,15 +1,15 @@
 import io
-import json
+
+from flask import send_file
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from flask import send_file
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.tools_manage_service import ToolManageService
-from werkzeug.exceptions import Forbidden
 
 
 class ToolProviderListApi(Resource):

api/controllers/console/workspace/workspace.py

@@ -1,18 +1,23 @@
 # -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse
+
 import services
 from controllers.console import api
 from controllers.console.admin import admin_required
-from controllers.console.datasets.error import (FileTooLargeError, NoFileUploadedError, TooManyFilesError,
-                                                UnsupportedFileTypeError)
+from controllers.console.datasets.error import (
+    FileTooLargeError,
+    NoFileUploadedError,
+    TooManyFilesError,
+    UnsupportedFileTypeError,
+)
 from controllers.console.error import AccountNotLinkTenantError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse
 from libs.helper import TimestampField
 from libs.login import login_required
 from models.account import Tenant

api/controllers/console/wraps.py

@@ -2,9 +2,10 @@
 import json
 from functools import wraps
 
-from controllers.console.workspace.error import AccountNotInitializedError
 from flask import abort, current_app, request
 from flask_login import current_user
+
+from controllers.console.workspace.error import AccountNotInitializedError
 from services.feature_service import FeatureService
 from services.operation_service import OperationService
 

api/controllers/files/image_preview.py

@@ -1,11 +1,12 @@
-import services
-from controllers.files import api
 from flask import Response, request
 from flask_restful import Resource
+from werkzeug.exceptions import NotFound
+
+import services
+from controllers.files import api
 from libs.exception import BaseHTTPException
 from services.account_service import TenantService
 from services.file_service import FileService
-from werkzeug.exceptions import NotFound
 
 
 class ImagePreviewApi(Resource):

api/controllers/files/tool_files.py

@@ -1,10 +1,11 @@
-from controllers.files import api
-from core.tools.tool_file_manager import ToolFileManager
 from flask import Response
 from flask_restful import Resource, reqparse
-from libs.exception import BaseHTTPException
 from werkzeug.exceptions import Forbidden, NotFound
 
+from controllers.files import api
+from core.tools.tool_file_manager import ToolFileManager
+from libs.exception import BaseHTTPException
+
 
 class ToolFilePreviewApi(Resource):
     def get(self, file_id, extension):

api/controllers/service_api/app/app.py

@@ -1,11 +1,12 @@
 # -*- coding:utf-8 -*-
 import json
 
+from flask import current_app
+from flask_restful import fields, marshal_with
+
 from controllers.service_api import api
 from controllers.service_api.wraps import AppApiResource
 from extensions.ext_database import db
-from flask import current_app
-from flask_restful import fields, marshal_with
 from models.model import App, AppModelConfig
 from models.tools import ApiToolProvider
 

api/controllers/service_api/app/audio.py

@@ -1,21 +1,33 @@
 import logging
 
+from flask import request
+from flask_restful import reqparse
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.service_api import api
-from controllers.service_api.app.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                               NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                               ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                               ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.service_api.app.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
 from controllers.service_api.wraps import AppApiResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
-from flask_restful import reqparse
 from models.model import App, AppModelConfig
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class AudioApi(AppApiResource):

api/controllers/service_api/app/completion.py

@@ -2,22 +2,29 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_restful import reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.service_api import api
 from controllers.service_api.app import create_or_update_end_user_for_user_id
-from controllers.service_api.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                               NotChatAppError, ProviderModelCurrentlyNotSupportError,
-                                               ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.service_api.app.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    NotChatAppError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.service_api.wraps import AppApiResource
 from core.application_queue_manager import ApplicationQueueManager
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import Response, stream_with_context
-from flask_restful import reqparse
 from libs.helper import uuid_value
 from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 class CompletionApi(AppApiResource):

api/controllers/service_api/app/conversation.py

@@ -1,16 +1,17 @@
 # -*- coding:utf-8 -*-
+from flask import request
+from flask_restful import marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 import services
 from controllers.service_api import api
 from controllers.service_api.app import create_or_update_end_user_for_user_id
 from controllers.service_api.app.error import NotChatAppError
 from controllers.service_api.wraps import AppApiResource
 from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
-from flask import request
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
+from libs.helper import uuid_value
 from services.conversation_service import ConversationService
-from werkzeug.exceptions import NotFound
 
 
 class ConversationApi(AppApiResource):

api/controllers/service_api/app/file.py

@@ -1,12 +1,17 @@
+from flask import request
+from flask_restful import marshal_with
+
 import services
 from controllers.service_api import api
 from controllers.service_api.app import create_or_update_end_user_for_user_id
-from controllers.service_api.app.error import (FileTooLargeError, NoFileUploadedError, TooManyFilesError,
-                                               UnsupportedFileTypeError)
+from controllers.service_api.app.error import (
+    FileTooLargeError,
+    NoFileUploadedError,
+    TooManyFilesError,
+    UnsupportedFileTypeError,
+)
 from controllers.service_api.wraps import AppApiResource
 from fields.file_fields import file_fields
-from flask import request
-from flask_restful import marshal_with
 from services.file_service import FileService
 
 

api/controllers/service_api/app/message.py

@@ -1,4 +1,8 @@
 # -*- coding:utf-8 -*-
+from flask_restful import fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 import services
 from controllers.service_api import api
 from controllers.service_api.app import create_or_update_end_user_for_user_id
@@ -6,12 +10,9 @@ from controllers.service_api.app.error import NotChatAppError
 from controllers.service_api.wraps import AppApiResource
 from extensions.ext_database import db
 from fields.conversation_fields import message_file_fields
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import TimestampField, uuid_value
 from models.model import EndUser, Message
 from services.message_service import MessageService
-from werkzeug.exceptions import NotFound
 
 
 class MessageListApi(AppApiResource):

api/controllers/service_api/dataset/dataset.py

@@ -1,3 +1,6 @@
+from flask import request
+from flask_restful import marshal, reqparse
+
 import services.dataset_service
 from controllers.service_api import api
 from controllers.service_api.dataset.error import DatasetNameDuplicateError
@@ -5,8 +8,6 @@ from controllers.service_api.wraps import DatasetApiResource
 from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from fields.dataset_fields import dataset_detail_fields
-from flask import request
-from flask_restful import marshal, reqparse
 from libs.login import current_user
 from models.dataset import Dataset
 from services.dataset_service import DatasetService

api/controllers/service_api/dataset/document.py

@@ -1,23 +1,28 @@
 import json
 
+from flask import request
+from flask_login import current_user
+from flask_restful import marshal, reqparse
+from sqlalchemy import desc
+from werkzeug.exceptions import NotFound
+
 import services.dataset_service
 from controllers.service_api import api
 from controllers.service_api.app.error import ProviderNotInitializeError
-from controllers.service_api.dataset.error import (ArchivedDocumentImmutableError, DocumentIndexingError,
-                                                   NoFileUploadedError, TooManyFilesError)
+from controllers.service_api.dataset.error import (
+    ArchivedDocumentImmutableError,
+    DocumentIndexingError,
+    NoFileUploadedError,
+    TooManyFilesError,
+)
 from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_resource_check
 from core.errors.error import ProviderTokenNotInitError
 from extensions.ext_database import db
 from fields.document_fields import document_fields, document_status_fields
-from flask import request
-from flask_login import current_user
-from flask_restful import marshal, reqparse
 from libs.login import current_user
 from models.dataset import Dataset, Document, DocumentSegment
 from services.dataset_service import DocumentService
 from services.file_service import FileService
-from sqlalchemy import desc
-from werkzeug.exceptions import NotFound
 
 
 class DocumentAddByTextApi(DatasetApiResource):

api/controllers/service_api/dataset/segment.py

@@ -1,3 +1,7 @@
+from flask_login import current_user
+from flask_restful import marshal, reqparse
+from werkzeug.exceptions import NotFound
+
 from controllers.service_api import api
 from controllers.service_api.app.error import ProviderNotInitializeError
 from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_resource_check
@@ -6,11 +10,8 @@ from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from extensions.ext_database import db
 from fields.segment_fields import segment_fields
-from flask_login import current_user
-from flask_restful import marshal, reqparse
 from models.dataset import Dataset, DocumentSegment
 from services.dataset_service import DatasetService, DocumentService, SegmentService
-from werkzeug.exceptions import NotFound
 
 
 class SegmentApi(DatasetApiResource):

api/controllers/service_api/index.py

@@ -1,7 +1,8 @@
-from controllers.service_api import api
 from flask import current_app
 from flask_restful import Resource
 
+from controllers.service_api import api
+
 
 class IndexApi(Resource):
     def get(self):

api/controllers/service_api/wraps.py

@@ -2,15 +2,16 @@
 from datetime import datetime
 from functools import wraps
 
-from extensions.ext_database import db
 from flask import current_app, request
 from flask_login import user_logged_in
 from flask_restful import Resource
+from werkzeug.exceptions import NotFound, Unauthorized
+
+from extensions.ext_database import db
 from libs.login import _get_user
 from models.account import Account, Tenant, TenantAccountJoin
 from models.model import ApiToken, App
 from services.feature_service import FeatureService
-from werkzeug.exceptions import NotFound, Unauthorized
 
 
 def validate_app_token(view=None):

api/controllers/web/app.py

@@ -1,11 +1,12 @@
 # -*- coding:utf-8 -*-
 import json
 
+from flask import current_app
+from flask_restful import fields, marshal_with
+
 from controllers.web import api
 from controllers.web.wraps import WebApiResource
 from extensions.ext_database import db
-from flask import current_app
-from flask_restful import fields, marshal_with
 from models.model import App, AppModelConfig
 from models.tools import ApiToolProvider
 

api/controllers/web/audio.py

@@ -1,21 +1,33 @@
 # -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.web import api
-from controllers.web.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                   NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                   ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                   ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.web.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
 from controllers.web.wraps import WebApiResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
 from models.model import App, AppModelConfig
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class AudioApi(WebApiResource):

api/controllers/web/completion.py

@@ -3,21 +3,29 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_restful import reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.web import api
-from controllers.web.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                   NotChatAppError, NotCompletionAppError, ProviderModelCurrentlyNotSupportError,
-                                   ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.web.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    NotChatAppError,
+    NotCompletionAppError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.web.wraps import WebApiResource
 from core.application_queue_manager import ApplicationQueueManager
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import Response, stream_with_context
-from flask_restful import reqparse
 from libs.helper import uuid_value
 from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 # define completion api for user

api/controllers/web/conversation.py

@@ -1,15 +1,16 @@
 # -*- coding:utf-8 -*-
+from flask_restful import marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 from controllers.web import api
 from controllers.web.error import NotChatAppError
 from controllers.web.wraps import WebApiResource
 from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
+from libs.helper import uuid_value
 from services.conversation_service import ConversationService
 from services.errors.conversation import ConversationNotExistsError, LastConversationNotExistsError
 from services.web_conversation_service import WebConversationService
-from werkzeug.exceptions import NotFound
 
 
 class ConversationListApi(WebApiResource):

api/controllers/web/file.py

@@ -1,10 +1,11 @@
+from flask import request
+from flask_restful import marshal_with
+
 import services
 from controllers.web import api
 from controllers.web.error import FileTooLargeError, NoFileUploadedError, TooManyFilesError, UnsupportedFileTypeError
 from controllers.web.wraps import WebApiResource
 from fields.file_fields import file_fields
-from flask import request
-from flask_restful import marshal_with
 from services.file_service import FileService
 
 

api/controllers/web/message.py

@@ -3,28 +3,35 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_restful import fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.web import api
-from controllers.web.error import (AppMoreLikeThisDisabledError, AppSuggestedQuestionsAfterAnswerDisabledError,
-                                   CompletionRequestError, NotChatAppError, NotCompletionAppError,
-                                   ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                   ProviderQuotaExceededError)
+from controllers.web.error import (
+    AppMoreLikeThisDisabledError,
+    AppSuggestedQuestionsAfterAnswerDisabledError,
+    CompletionRequestError,
+    NotChatAppError,
+    NotCompletionAppError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.web.wraps import WebApiResource
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from fields.conversation_fields import message_file_fields
 from fields.message_fields import agent_thought_fields
-from flask import Response, stream_with_context
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import TimestampField, uuid_value
 from services.completion_service import CompletionService
 from services.errors.app import MoreLikeThisDisabledError
 from services.errors.conversation import ConversationNotExistsError
 from services.errors.message import MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError
 from services.message_service import MessageService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 class MessageListApi(WebApiResource):

api/controllers/web/passport.py

@@ -1,13 +1,14 @@
 # -*- coding:utf-8 -*-
 import uuid
 
-from controllers.web import api
-from extensions.ext_database import db
 from flask import request
 from flask_restful import Resource
+from werkzeug.exceptions import NotFound, Unauthorized
+
+from controllers.web import api
+from extensions.ext_database import db
 from libs.passport import PassportService
 from models.model import App, EndUser, Site
-from werkzeug.exceptions import NotFound, Unauthorized
 
 
 class PassportResource(Resource):

api/controllers/web/saved_message.py

@@ -1,13 +1,14 @@
+from flask_restful import fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 from controllers.web import api
 from controllers.web.error import NotCompletionAppError
 from controllers.web.wraps import WebApiResource
 from fields.conversation_fields import message_file_fields
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import TimestampField, uuid_value
 from services.errors.message import MessageNotExistsError
 from services.saved_message_service import SavedMessageService
-from werkzeug.exceptions import NotFound
 
 feedback_fields = {
     'rating': fields.String

api/controllers/web/site.py

@@ -1,14 +1,14 @@
 # -*- coding:utf-8 -*-
-import os
+
+from flask import current_app
+from flask_restful import fields, marshal_with
+from werkzeug.exceptions import Forbidden
 
 from controllers.web import api
 from controllers.web.wraps import WebApiResource
 from extensions.ext_database import db
-from flask import current_app
-from flask_restful import fields, marshal_with
 from models.model import Site
 from services.feature_service import FeatureService
-from werkzeug.exceptions import Forbidden
 
 
 class AppSiteApi(WebApiResource):

api/controllers/web/wraps.py

@@ -1,12 +1,13 @@
 # -*- coding:utf-8 -*-
 from functools import wraps
 
-from extensions.ext_database import db
 from flask import request
 from flask_restful import Resource
+from werkzeug.exceptions import NotFound, Unauthorized
+
+from extensions.ext_database import db
 from libs.passport import PassportService
 from models.model import App, EndUser, Site
-from werkzeug.exceptions import NotFound, Unauthorized
 
 
 def validate_jwt_token(view=None):

api/core/agent/agent/calc_token_mixin.py

@@ -1,11 +1,9 @@
 from typing import List, cast
 
 from core.entities.application_entities import ModelConfigEntity
-from core.entities.message_entities import lc_messages_to_prompt_messages
 from core.model_runtime.entities.message_entities import PromptMessage
 from core.model_runtime.entities.model_entities import ModelPropertyKey
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from langchain.schema import BaseMessage
 
 
 class CalcTokenMixin:

api/core/agent/agent/multi_dataset_router_agent.py

@@ -1,10 +1,5 @@
-from typing import Any, List, Optional, Sequence, Tuple, Union, cast
+from typing import Any, List, Optional, Sequence, Tuple, Union
 
-from core.entities.application_entities import ModelConfigEntity
-from core.entities.message_entities import lc_messages_to_prompt_messages
-from core.model_manager import ModelInstance
-from core.model_runtime.entities.message_entities import PromptMessageTool
-from core.third_party.langchain.llms.fake import FakeLLM
 from langchain.agents import BaseSingleActionAgent, OpenAIFunctionsAgent
 from langchain.agents.openai_functions_agent.base import _format_intermediate_steps, _parse_ai_message
 from langchain.callbacks.base import BaseCallbackManager
@@ -14,6 +9,12 @@ from langchain.schema import AgentAction, AgentFinish, AIMessage, SystemMessage
 from langchain.tools import BaseTool
 from pydantic import root_validator
 
+from core.entities.application_entities import ModelConfigEntity
+from core.entities.message_entities import lc_messages_to_prompt_messages
+from core.model_manager import ModelInstance
+from core.model_runtime.entities.message_entities import PromptMessageTool
+from core.third_party.langchain.llms.fake import FakeLLM
+
 
 class MultiDatasetRouterAgent(OpenAIFunctionsAgent):
     """

api/core/agent/agent/openai_function_call.py

@@ -1,4 +1,23 @@
-from typing import Any, List, Optional, Sequence, Tuple, Union, cast
+from typing import Any, List, Optional, Sequence, Tuple, Union
+
+from langchain.agents import BaseSingleActionAgent, OpenAIFunctionsAgent
+from langchain.agents.openai_functions_agent.base import _format_intermediate_steps, _parse_ai_message
+from langchain.callbacks.base import BaseCallbackManager
+from langchain.callbacks.manager import Callbacks
+from langchain.chat_models.openai import _convert_message_to_dict, _import_tiktoken
+from langchain.memory.prompt import SUMMARY_PROMPT
+from langchain.prompts.chat import BaseMessagePromptTemplate
+from langchain.schema import (
+    AgentAction,
+    AgentFinish,
+    AIMessage,
+    BaseMessage,
+    HumanMessage,
+    SystemMessage,
+    get_buffer_string,
+)
+from langchain.tools import BaseTool
+from pydantic import root_validator
 
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.agent.agent.calc_token_mixin import CalcTokenMixin, ExceededLLMTokensLimitError
@@ -7,19 +26,7 @@ from core.entities.application_entities import ModelConfigEntity
 from core.entities.message_entities import lc_messages_to_prompt_messages
 from core.model_manager import ModelInstance
 from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool
-from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.third_party.langchain.llms.fake import FakeLLM
-from langchain.agents import BaseSingleActionAgent, OpenAIFunctionsAgent
-from langchain.agents.openai_functions_agent.base import _format_intermediate_steps, _parse_ai_message
-from langchain.callbacks.base import BaseCallbackManager
-from langchain.callbacks.manager import Callbacks
-from langchain.chat_models.openai import _convert_message_to_dict, _import_tiktoken
-from langchain.memory.prompt import SUMMARY_PROMPT
-from langchain.prompts.chat import BaseMessagePromptTemplate
-from langchain.schema import (AgentAction, AgentFinish, AIMessage, BaseMessage, HumanMessage, SystemMessage,
-                              get_buffer_string)
-from langchain.tools import BaseTool
-from pydantic import root_validator
 
 
 class AutoSummarizingOpenAIFunctionCallAgent(OpenAIFunctionsAgent, CalcTokenMixin):

api/core/agent/agent/structed_multi_dataset_router_agent.py

@@ -1,8 +1,6 @@
 import re
 from typing import Any, List, Optional, Sequence, Tuple, Union, cast
 
-from core.chain.llm_chain import LLMChain
-from core.entities.application_entities import ModelConfigEntity
 from langchain import BasePromptTemplate, PromptTemplate
 from langchain.agents import Agent, AgentOutputParser, StructuredChatAgent
 from langchain.agents.structured_chat.base import HUMAN_MESSAGE_TEMPLATE
@@ -13,6 +11,9 @@ from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, Sy
 from langchain.schema import AgentAction, AgentFinish, OutputParserException
 from langchain.tools import BaseTool
 
+from core.chain.llm_chain import LLMChain
+from core.entities.application_entities import ModelConfigEntity
+
 FORMAT_INSTRUCTIONS = """Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input).
 The nouns in the format of "Thought", "Action", "Action Input", "Final Answer" must be expressed in English.
 Valid "action" values: "Final Answer" or {tool_names}

api/core/agent/agent/structured_chat.py

@@ -1,11 +1,6 @@
 import re
 from typing import Any, List, Optional, Sequence, Tuple, Union, cast
 
-from core.agent.agent.agent_llm_callback import AgentLLMCallback
-from core.agent.agent.calc_token_mixin import CalcTokenMixin, ExceededLLMTokensLimitError
-from core.chain.llm_chain import LLMChain
-from core.entities.application_entities import ModelConfigEntity
-from core.entities.message_entities import lc_messages_to_prompt_messages
 from langchain import BasePromptTemplate, PromptTemplate
 from langchain.agents import Agent, AgentOutputParser, StructuredChatAgent
 from langchain.agents.structured_chat.base import HUMAN_MESSAGE_TEMPLATE
@@ -14,10 +9,23 @@ from langchain.callbacks.base import BaseCallbackManager
 from langchain.callbacks.manager import Callbacks
 from langchain.memory.prompt import SUMMARY_PROMPT
 from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate
-from langchain.schema import (AgentAction, AgentFinish, AIMessage, BaseMessage, HumanMessage, OutputParserException,
-                              get_buffer_string)
+from langchain.schema import (
+    AgentAction,
+    AgentFinish,
+    AIMessage,
+    BaseMessage,
+    HumanMessage,
+    OutputParserException,
+    get_buffer_string,
+)
 from langchain.tools import BaseTool
 
+from core.agent.agent.agent_llm_callback import AgentLLMCallback
+from core.agent.agent.calc_token_mixin import CalcTokenMixin, ExceededLLMTokensLimitError
+from core.chain.llm_chain import LLMChain
+from core.entities.application_entities import ModelConfigEntity
+from core.entities.message_entities import lc_messages_to_prompt_messages
+
 FORMAT_INSTRUCTIONS = """Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input).
 The nouns in the format of "Thought", "Action", "Action Input", "Final Answer" must be expressed in English.
 Valid "action" values: "Final Answer" or {tool_names}

api/core/agent/agent_executor.py

@@ -2,6 +2,12 @@ import enum
 import logging
 from typing import Optional, Union
 
+from langchain.agents import AgentExecutor as LCAgentExecutor
+from langchain.agents import BaseMultiActionAgent, BaseSingleActionAgent
+from langchain.callbacks.manager import Callbacks
+from langchain.tools import BaseTool
+from pydantic import BaseModel, Extra
+
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.agent.agent.multi_dataset_router_agent import MultiDatasetRouterAgent
 from core.agent.agent.openai_function_call import AutoSummarizingOpenAIFunctionCallAgent
@@ -15,11 +21,6 @@ from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.errors.invoke import InvokeError
 from core.tools.tool.dataset_retriever.dataset_multi_retriever_tool import DatasetMultiRetrieverTool
 from core.tools.tool.dataset_retriever.dataset_retriever_tool import DatasetRetrieverTool
-from langchain.agents import AgentExecutor as LCAgentExecutor
-from langchain.agents import BaseMultiActionAgent, BaseSingleActionAgent
-from langchain.callbacks.manager import Callbacks
-from langchain.tools import BaseTool
-from pydantic import BaseModel, Extra
 
 
 class PlanningStrategy(str, enum.Enum):

api/core/app_runner/app_runner.py

@@ -2,9 +2,14 @@ import time
 from typing import Generator, List, Optional, Tuple, Union, cast
 
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
-from core.entities.application_entities import (ApplicationGenerateEntity, AppOrchestrationConfigEntity,
-                                                ExternalDataVariableEntity, InvokeFrom, ModelConfigEntity,
-                                                PromptTemplateEntity)
+from core.entities.application_entities import (
+    ApplicationGenerateEntity,
+    AppOrchestrationConfigEntity,
+    ExternalDataVariableEntity,
+    InvokeFrom,
+    ModelConfigEntity,
+    PromptTemplateEntity,
+)
 from core.features.annotation_reply import AnnotationReplyFeature
 from core.features.external_data_fetch import ExternalDataFetchFeature
 from core.features.hosting_moderation import HostingModerationFeature

api/core/app_runner/generate_task_pipeline.py

@@ -3,28 +3,42 @@ import logging
 import time
 from typing import Generator, Optional, Union, cast
 
+from pydantic import BaseModel
+
 from core.app_runner.moderation_handler import ModerationRule, OutputModerationHandler
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.entities.application_entities import ApplicationGenerateEntity, InvokeFrom
-from core.entities.queue_entities import (AnnotationReplyEvent, QueueAgentMessageEvent, QueueAgentThoughtEvent,
-                                          QueueErrorEvent, QueueMessageEndEvent, QueueMessageEvent,
-                                          QueueMessageFileEvent, QueueMessageReplaceEvent, QueuePingEvent,
-                                          QueueRetrieverResourcesEvent, QueueStopEvent)
+from core.entities.queue_entities import (
+    AnnotationReplyEvent,
+    QueueAgentMessageEvent,
+    QueueAgentThoughtEvent,
+    QueueErrorEvent,
+    QueueMessageEndEvent,
+    QueueMessageEvent,
+    QueueMessageFileEvent,
+    QueueMessageReplaceEvent,
+    QueuePingEvent,
+    QueueRetrieverResourcesEvent,
+    QueueStopEvent,
+)
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
-from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent,
-                                                          PromptMessage, PromptMessageContentType, PromptMessageRole,
-                                                          TextPromptMessageContent)
+from core.model_runtime.entities.message_entities import (
+    AssistantPromptMessage,
+    ImagePromptMessageContent,
+    PromptMessage,
+    PromptMessageContentType,
+    PromptMessageRole,
+    TextPromptMessageContent,
+)
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.utils.encoders import jsonable_encoder
 from core.prompt.prompt_template import PromptTemplateParser
 from core.tools.tool_file_manager import ToolFileManager
-from core.tools.tool_manager import ToolManager
 from events.message_event import message_was_created
 from extensions.ext_database import db
 from models.model import Conversation, Message, MessageAgentThought, MessageFile
-from pydantic import BaseModel
 from services.annotation_service import AppAnnotationService
 
 logger = logging.getLogger(__name__)

api/core/app_runner/moderation_handler.py

@@ -3,11 +3,12 @@ import threading
 import time
 from typing import Any, Dict, Optional
 
+from flask import Flask, current_app
+from pydantic import BaseModel
+
 from core.application_queue_manager import PublishFrom
 from core.moderation.base import ModerationAction, ModerationOutputsResult
 from core.moderation.factory import ModerationFactory
-from flask import Flask, current_app
-from pydantic import BaseModel
 
 logger = logging.getLogger(__name__)
 

api/core/application_manager.py

@@ -4,17 +4,30 @@ import threading
 import uuid
 from typing import Any, Generator, Optional, Tuple, Union, cast
 
+from flask import Flask, current_app
+from pydantic import ValidationError
+
 from core.app_runner.assistant_app_runner import AssistantApplicationRunner
 from core.app_runner.basic_app_runner import BasicApplicationRunner
 from core.app_runner.generate_task_pipeline import GenerateTaskPipeline
 from core.application_queue_manager import ApplicationQueueManager, ConversationTaskStoppedException, PublishFrom
-from core.entities.application_entities import (AdvancedChatPromptTemplateEntity,
-                                                AdvancedCompletionPromptTemplateEntity, AgentEntity, AgentPromptEntity,
-                                                AgentToolEntity, ApplicationGenerateEntity,
-                                                AppOrchestrationConfigEntity, DatasetEntity,
-                                                DatasetRetrieveConfigEntity, ExternalDataVariableEntity,
-                                                FileUploadEntity, InvokeFrom, ModelConfigEntity, PromptTemplateEntity,
-                                                SensitiveWordAvoidanceEntity)
+from core.entities.application_entities import (
+    AdvancedChatPromptTemplateEntity,
+    AdvancedCompletionPromptTemplateEntity,
+    AgentEntity,
+    AgentPromptEntity,
+    AgentToolEntity,
+    ApplicationGenerateEntity,
+    AppOrchestrationConfigEntity,
+    DatasetEntity,
+    DatasetRetrieveConfigEntity,
+    ExternalDataVariableEntity,
+    FileUploadEntity,
+    InvokeFrom,
+    ModelConfigEntity,
+    PromptTemplateEntity,
+    SensitiveWordAvoidanceEntity,
+)
 from core.entities.model_entities import ModelStatus
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.file.file_obj import FileObj
@@ -26,10 +39,8 @@ from core.prompt.prompt_template import PromptTemplateParser
 from core.provider_manager import ProviderManager
 from core.tools.prompt.template import REACT_PROMPT_TEMPLATES
 from extensions.ext_database import db
-from flask import Flask, current_app
 from models.account import Account
 from models.model import App, Conversation, EndUser, Message, MessageFile
-from pydantic import ValidationError
 
 logger = logging.getLogger(__name__)
 

api/core/application_queue_manager.py

@@ -3,15 +3,27 @@ import time
 from enum import Enum
 from typing import Any, Generator
 
+from sqlalchemy.orm import DeclarativeMeta
+
 from core.entities.application_entities import InvokeFrom
-from core.entities.queue_entities import (AnnotationReplyEvent, AppQueueEvent, QueueAgentMessageEvent,
-                                          QueueAgentThoughtEvent, QueueErrorEvent, QueueMessage, QueueMessageEndEvent,
-                                          QueueMessageEvent, QueueMessageFileEvent, QueueMessageReplaceEvent,
-                                          QueuePingEvent, QueueRetrieverResourcesEvent, QueueStopEvent)
+from core.entities.queue_entities import (
+    AnnotationReplyEvent,
+    AppQueueEvent,
+    QueueAgentMessageEvent,
+    QueueAgentThoughtEvent,
+    QueueErrorEvent,
+    QueueMessage,
+    QueueMessageEndEvent,
+    QueueMessageEvent,
+    QueueMessageFileEvent,
+    QueueMessageReplaceEvent,
+    QueuePingEvent,
+    QueueRetrieverResourcesEvent,
+    QueueStopEvent,
+)
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
 from extensions.ext_redis import redis_client
 from models.model import MessageAgentThought, MessageFile
-from sqlalchemy.orm import DeclarativeMeta
 
 
 class PublishFrom(Enum):

api/core/callback_handler/agent_loop_gather_callback_handler.py

@@ -3,6 +3,10 @@ import logging
 import time
 from typing import Any, Dict, List, Optional, Union, cast
 
+from langchain.agents import openai_functions_agent, openai_functions_multi_agent
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.schema import AgentAction, AgentFinish, BaseMessage, LLMResult
+
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.callback_handler.entity.agent_loop import AgentLoop
 from core.entities.application_entities import ModelConfigEntity
@@ -10,9 +14,6 @@ from core.model_runtime.entities.llm_entities import LLMResult as RuntimeLLMResu
 from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from extensions.ext_database import db
-from langchain.agents import openai_functions_agent, openai_functions_multi_agent
-from langchain.callbacks.base import BaseCallbackHandler
-from langchain.schema import AgentAction, AgentFinish, BaseMessage, ChatGeneration, LLMResult
 from models.model import Message, MessageAgentThought, MessageChain
 
 

api/core/callback_handler/index_tool_callback_handler.py

@@ -1,9 +1,10 @@
-from typing import List, Union
+from typing import List
+
+from langchain.schema import Document
 
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.entities.application_entities import InvokeFrom
 from extensions.ext_database import db
-from langchain.schema import Document
 from models.dataset import DatasetQuery, DocumentSegment
 from models.model import DatasetRetrieverResource
 

api/core/chain/llm_chain.py

@@ -1,14 +1,15 @@
 from typing import Any, Dict, List, Optional
 
+from langchain import LLMChain as LCLLMChain
+from langchain.callbacks.manager import CallbackManagerForChainRun
+from langchain.schema import Generation, LLMResult
+from langchain.schema.language_model import BaseLanguageModel
+
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.entities.application_entities import ModelConfigEntity
 from core.entities.message_entities import lc_messages_to_prompt_messages
 from core.model_manager import ModelInstance
 from core.third_party.langchain.llms.fake import FakeLLM
-from langchain import LLMChain as LCLLMChain
-from langchain.callbacks.manager import CallbackManagerForChainRun
-from langchain.schema import Generation, LLMResult
-from langchain.schema.language_model import BaseLanguageModel
 
 
 class LLMChain(LCLLMChain):

api/core/data_loader/file_extractor.py

@@ -3,6 +3,10 @@ from pathlib import Path
 from typing import List, Optional, Union
 
 import requests
+from flask import current_app
+from langchain.document_loaders import Docx2txtLoader, TextLoader
+from langchain.schema import Document
+
 from core.data_loader.loader.csv_loader import CSVLoader
 from core.data_loader.loader.excel import ExcelLoader
 from core.data_loader.loader.html import HTMLLoader
@@ -16,9 +20,6 @@ from core.data_loader.loader.unstructured.unstructured_pptx import UnstructuredP
 from core.data_loader.loader.unstructured.unstructured_text import UnstructuredTextLoader
 from core.data_loader.loader.unstructured.unstructured_xml import UnstructuredXmlLoader
 from extensions.ext_storage import storage
-from flask import current_app
-from langchain.document_loaders import Docx2txtLoader, TextLoader
-from langchain.schema import Document
 from models.model import UploadFile
 
 SUPPORT_URL_CONTENT_TYPES = ['application/pdf', 'text/plain']

api/core/data_loader/loader/excel.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from typing import List
 

api/core/data_loader/loader/notion.py

@@ -3,10 +3,11 @@ import logging
 from typing import Any, Dict, List, Optional
 
 import requests
-from extensions.ext_database import db
 from flask import current_app
 from langchain.document_loaders.base import BaseLoader
 from langchain.schema import Document
+
+from extensions.ext_database import db
 from models.dataset import Document as DocumentModel
 from models.source import DataSourceBinding
 

api/core/data_loader/loader/pdf.py

@@ -1,10 +1,11 @@
 import logging
 from typing import List, Optional
 
-from extensions.ext_storage import storage
 from langchain.document_loaders import PyPDFium2Loader
 from langchain.document_loaders.base import BaseLoader
 from langchain.schema import Document
+
+from extensions.ext_storage import storage
 from models.model import UploadFile
 
 logger = logging.getLogger(__name__)

api/core/docstore/dataset_docstore.py

@@ -1,12 +1,13 @@
 from typing import Any, Dict, Optional, Sequence, cast
 
+from langchain.schema import Document
+from sqlalchemy import func
+
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
 from extensions.ext_database import db
-from langchain.schema import Document
 from models.dataset import Dataset, DocumentSegment
-from sqlalchemy import func
 
 
 class DatasetDocumentStore:

api/core/embedding/cached_embedding.py

@@ -1,18 +1,17 @@
 import base64
-import json
 import logging
 from typing import List, Optional, cast
 
 import numpy as np
+from langchain.embeddings.base import Embeddings
+from sqlalchemy.exc import IntegrityError
+
 from core.model_manager import ModelInstance
 from core.model_runtime.entities.model_entities import ModelPropertyKey
 from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from langchain.embeddings.base import Embeddings
 from libs import helper
-from models.dataset import Embedding
-from sqlalchemy.exc import IntegrityError
 
 logger = logging.getLogger(__name__)
 

api/core/entities/application_entities.py

@@ -1,11 +1,12 @@
 from enum import Enum
-from typing import Any, Literal, Optional, Union, cast
+from typing import Any, Literal, Optional, Union
+
+from pydantic import BaseModel
 
 from core.entities.provider_configuration import ProviderModelBundle
 from core.file.file_obj import FileObj
 from core.model_runtime.entities.message_entities import PromptMessageRole
 from core.model_runtime.entities.model_entities import AIModelEntity
-from pydantic import BaseModel
 
 
 class ModelConfigEntity(BaseModel):

api/core/entities/message_entities.py

@@ -1,12 +1,19 @@
 import enum
 from typing import Any, cast
 
-from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent,
-                                                          PromptMessage, SystemPromptMessage, TextPromptMessageContent,
-                                                          ToolPromptMessage, UserPromptMessage)
 from langchain.schema import AIMessage, BaseMessage, FunctionMessage, HumanMessage, SystemMessage
 from pydantic import BaseModel
 
+from core.model_runtime.entities.message_entities import (
+    AssistantPromptMessage,
+    ImagePromptMessageContent,
+    PromptMessage,
+    SystemPromptMessage,
+    TextPromptMessageContent,
+    ToolPromptMessage,
+    UserPromptMessage,
+)
+
 
 class PromptMessageFileType(enum.Enum):
     IMAGE = 'image'

api/core/entities/model_entities.py

@@ -1,10 +1,11 @@
 from enum import Enum
 from typing import Optional
 
+from pydantic import BaseModel
+
 from core.model_runtime.entities.common_entities import I18nObject
 from core.model_runtime.entities.model_entities import ModelType, ProviderModel
-from core.model_runtime.entities.provider_entities import ProviderEntity, SimpleProviderEntity
-from pydantic import BaseModel
+from core.model_runtime.entities.provider_entities import ProviderEntity
 
 
 class ModelStatus(Enum):

api/core/entities/provider_configuration.py

@@ -4,20 +4,24 @@ import logging
 from json import JSONDecodeError
 from typing import Dict, Iterator, List, Optional, Tuple
 
+from pydantic import BaseModel
+
 from core.entities.model_entities import ModelStatus, ModelWithProviderEntity, SimpleModelProviderEntity
 from core.entities.provider_entities import CustomConfiguration, SystemConfiguration, SystemConfigurationStatus
 from core.helper import encrypter
 from core.helper.model_provider_cache import ProviderCredentialsCache, ProviderCredentialsCacheType
 from core.model_runtime.entities.model_entities import FetchFrom, ModelType
-from core.model_runtime.entities.provider_entities import (ConfigurateMethod, CredentialFormSchema, FormType,
-                                                           ProviderEntity)
+from core.model_runtime.entities.provider_entities import (
+    ConfigurateMethod,
+    CredentialFormSchema,
+    FormType,
+    ProviderEntity,
+)
 from core.model_runtime.model_providers import model_provider_factory
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 from core.model_runtime.model_providers.__base.model_provider import ModelProvider
-from core.model_runtime.utils import encoders
 from extensions.ext_database import db
 from models.provider import Provider, ProviderModel, ProviderType, TenantPreferredModelProvider
-from pydantic import BaseModel
 
 logger = logging.getLogger(__name__)
 

api/core/entities/provider_entities.py

@@ -1,9 +1,10 @@
 from enum import Enum
 from typing import Optional
 
+from pydantic import BaseModel
+
 from core.model_runtime.entities.model_entities import ModelType
 from models.provider import ProviderQuotaType
-from pydantic import BaseModel
 
 
 class QuotaUnit(Enum):

api/core/entities/queue_entities.py

@@ -1,9 +1,10 @@
 from enum import Enum
 from typing import Any
 
-from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
 from pydantic import BaseModel
 
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
+
 
 class QueueEvent(Enum):
     """

api/core/extension/api_based_extension_requestor.py

@@ -1,6 +1,7 @@
 import os
 
 import requests
+
 from models.api_based_extension import APIBasedExtensionPoint
 
 

api/core/features/agent_runner.py

@@ -1,5 +1,7 @@
 import logging
-from typing import List, Optional, cast
+from typing import Optional, cast
+
+from langchain.tools import BaseTool
 
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.agent.agent_executor import AgentConfiguration, AgentExecutor, PlanningStrategy
@@ -7,20 +9,20 @@ from core.application_queue_manager import ApplicationQueueManager
 from core.callback_handler.agent_loop_gather_callback_handler import AgentLoopGatherCallbackHandler
 from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
 from core.callback_handler.std_out_callback_handler import DifyStdOutCallbackHandler
-from core.entities.application_entities import (AgentEntity, AgentToolEntity, AppOrchestrationConfigEntity, InvokeFrom,
-                                                ModelConfigEntity)
+from core.entities.application_entities import (
+    AgentEntity,
+    AppOrchestrationConfigEntity,
+    InvokeFrom,
+    ModelConfigEntity,
+)
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.entities.model_entities import ModelFeature, ModelType
 from core.model_runtime.model_providers import model_provider_factory
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.tools.tool.dataset_retriever.dataset_retriever_tool import DatasetRetrieverTool
 from extensions.ext_database import db
-from langchain import WikipediaAPIWrapper
-from langchain.callbacks.base import BaseCallbackHandler
-from langchain.tools import BaseTool, Tool, WikipediaQueryRun
 from models.dataset import Dataset
 from models.model import Message
-from pydantic import BaseModel, Field
 
 logger = logging.getLogger(__name__)