Austin/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2024-11-16 03:32:23 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

refactor: extract storage provider configs into dify configs (#5443)

Browse Source
This commit is contained in:
Bowen Liang 2024-06-22 10:07:03 +08:00 committed by GitHub
parent 8890978ad3
commit b05cc3a1e4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 178 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
api/config.py
View File

@ -9,8 +9,6 @@ DEFAULTS = {
'DB_PORT': '5432',
'DB_DATABASE': 'dify',
'DB_CHARSET': '',
'S3_USE_AWS_MANAGED_IAM': 'False',
'S3_ADDRESS_STYLE': 'auto',
'SQLALCHEMY_DATABASE_URI_SCHEME': 'postgresql',
'SQLALCHEMY_POOL_SIZE': 30,
'SQLALCHEMY_MAX_OVERFLOW': 10,
@ -100,41 +98,6 @@ class Config:
if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False
# S3 Storage settings
self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM')
self.S3_ENDPOINT = get_env('S3_ENDPOINT')
self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME')
self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY')
self.S3_SECRET_KEY = get_env('S3_SECRET_KEY')
self.S3_REGION = get_env('S3_REGION')
self.S3_ADDRESS_STYLE = get_env('S3_ADDRESS_STYLE')
# Azure Blob Storage settings
self.AZURE_BLOB_ACCOUNT_NAME = get_env('AZURE_BLOB_ACCOUNT_NAME')
self.AZURE_BLOB_ACCOUNT_KEY = get_env('AZURE_BLOB_ACCOUNT_KEY')
self.AZURE_BLOB_CONTAINER_NAME = get_env('AZURE_BLOB_CONTAINER_NAME')
self.AZURE_BLOB_ACCOUNT_URL = get_env('AZURE_BLOB_ACCOUNT_URL')
# Aliyun Storage settings
self.ALIYUN_OSS_BUCKET_NAME = get_env('ALIYUN_OSS_BUCKET_NAME')
self.ALIYUN_OSS_ACCESS_KEY = get_env('ALIYUN_OSS_ACCESS_KEY')
self.ALIYUN_OSS_SECRET_KEY = get_env('ALIYUN_OSS_SECRET_KEY')
self.ALIYUN_OSS_ENDPOINT = get_env('ALIYUN_OSS_ENDPOINT')
self.ALIYUN_OSS_REGION = get_env('ALIYUN_OSS_REGION')
self.ALIYUN_OSS_AUTH_VERSION = get_env('ALIYUN_OSS_AUTH_VERSION')
# Google Cloud Storage settings
self.GOOGLE_STORAGE_BUCKET_NAME = get_env('GOOGLE_STORAGE_BUCKET_NAME')
self.GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 = get_env('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64')
# Tencent Cos Storage settings
self.TENCENT_COS_BUCKET_NAME = get_env('TENCENT_COS_BUCKET_NAME')
self.TENCENT_COS_REGION = get_env('TENCENT_COS_REGION')
self.TENCENT_COS_SECRET_ID = get_env('TENCENT_COS_SECRET_ID')
self.TENCENT_COS_SECRET_KEY = get_env('TENCENT_COS_SECRET_KEY')
self.TENCENT_COS_SCHEME = get_env('TENCENT_COS_SCHEME')
# ------------------------
# Platform Configurations.
# ------------------------

14
api/configs/middleware/__init__.py
View File

@ -3,6 +3,11 @@ from typing import Optional
from pydantic import BaseModel, Field
from configs.middleware.redis_config import RedisConfig
from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
from configs.middleware.vdb.chroma_configs import ChromaConfigs
from configs.middleware.vdb.milvus_configs import MilvusConfigs
from configs.middleware.vdb.opensearch_configs import OpenSearchConfigs
@ -48,13 +53,21 @@ class MiddlewareConfig(
# place the configs in alphabet order
KeywordStoreConfigs,
RedisConfig,
# configs of storage and storage providers
StorageConfigs,
AliyunOSSStorageConfig,
AzureBlobStorageConfig,
GoogleCloudStorageConfig,
TencentCloudCOSStorageConfig,
S3StorageConfig,
# configs of vdb and vdb providers
VectorStoreConfigs,
ChromaConfigs,
MilvusConfigs,
OpenSearchConfigs,
OracleConfigs,
PGVectorConfigs,
PGVectoRSConfigs,
QdrantConfigs,
@ -62,6 +75,5 @@ class MiddlewareConfig(
TencentVectorDBConfigs,
TiDBVectorConfigs,
WeaviateConfigs,
OracleConfigs,
):
pass

39
api/configs/middleware/storage/aliyun_oss_storage_config.py Normal file
View File

@ -0,0 +1,39 @@
from typing import Optional
from pydantic import BaseModel, Field
class AliyunOSSStorageConfig(BaseModel):
"""
Aliyun storage configs
"""
ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
description='Aliyun storage ',
default=None,
)
ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
description='Aliyun storage access key',
default=None,
)
ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
description='Aliyun storage secret key',
default=None,
)
ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
description='Aliyun storage endpoint URL',
default=None,
)
ALIYUN_OSS_REGION: Optional[str] = Field(
description='Aliyun storage region',
default=None,
)
ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
description='Aliyun storage authentication version',
default=None,
)

44
api/configs/middleware/storage/amazon_s3_storage_config.py Normal file
View File

@ -0,0 +1,44 @@
from typing import Optional
from pydantic import BaseModel, Field
class S3StorageConfig(BaseModel):
"""
S3 storage configs
"""
S3_ENDPOINT: Optional[str] = Field(
description='S3 storage endpoint',
default=None,
)
S3_REGION: Optional[str] = Field(
description='S3 storage region',
default=None,
)
S3_BUCKET_NAME: Optional[str] = Field(
description='S3 storage bucket name',
default=None,
)
S3_ACCESS_KEY: Optional[str] = Field(
description='S3 storage access key',
default=None,
)
S3_SECRET_KEY: Optional[str] = Field(
description='S3 storage secret key',
default=None,
)
S3_ADDRESS_STYLE: str = Field(
description='S3 storage address style',
default='auto',
)
S3_USE_AWS_MANAGED_IAM: bool = Field(
description='whether to use aws managed IAM for S3',
default=False,
)

29
api/configs/middleware/storage/azure_blob_storage_config.py Normal file
View File

@ -0,0 +1,29 @@
from typing import Optional
from pydantic import BaseModel, Field
class AzureBlobStorageConfig(BaseModel):
"""
Azure Blob storage configs
"""
AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field(
description='Azure Blob account name',
default=None,
)
AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field(
description='Azure Blob account key',
default=None,
)
AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field(
description='Azure Blob container name',
default=None,
)
AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
description='Azure Blob account url',
default=None,
)

19
api/configs/middleware/storage/google_cloud_storage_config.py Normal file
View File

@ -0,0 +1,19 @@
from typing import Optional
from pydantic import BaseModel, Field
class GoogleCloudStorageConfig(BaseModel):
"""
Google Cloud storage configs
"""
GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field(
description='Google Cloud storage bucket name',
default=None,
)
GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field(
description='Google Cloud storage service account json base64',
default=None,
)

34
api/configs/middleware/storage/tencent_cos_storage_config.py Normal file
View File

@ -0,0 +1,34 @@
from typing import Optional
from pydantic import BaseModel, Field
class TencentCloudCOSStorageConfig(BaseModel):
"""
Tencent Cloud COS storage configs
"""
TENCENT_COS_BUCKET_NAME: Optional[str] = Field(
description='Tencent Cloud COS bucket name',
default=None,
)
TENCENT_COS_REGION: Optional[str] = Field(
description='Tencent Cloud COS region',
default=None,
)
TENCENT_COS_SECRET_ID: Optional[str] = Field(
description='Tencent Cloud COS secret id',
default=None,
)
TENCENT_COS_SECRET_KEY: Optional[str] = Field(
description='Tencent Cloud COS secret key',
default=None,
)
TENCENT_COS_SCHEME: Optional[str] = Field(
description='Tencent Cloud COS scheme',
default=None,
)