add volcengine tos storage (#8164)

This commit is contained in:
ice yao 2024-09-10 09:19:47 +08:00 committed by GitHub
parent 144d30d7ef
commit c8df92d0eb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 131 additions and 2 deletions

View File

@ -39,7 +39,7 @@ DB_DATABASE=dify
# Storage configuration # Storage configuration
# use for store upload files, private keys... # use for store upload files, private keys...
# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs # storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs, volcengine-tos
STORAGE_TYPE=local STORAGE_TYPE=local
STORAGE_LOCAL_PATH=storage STORAGE_LOCAL_PATH=storage
S3_USE_AWS_MANAGED_IAM=false S3_USE_AWS_MANAGED_IAM=false
@ -86,6 +86,13 @@ OCI_ACCESS_KEY=your-access-key
OCI_SECRET_KEY=your-secret-key OCI_SECRET_KEY=your-secret-key
OCI_REGION=your-region OCI_REGION=your-region
# Volcengine tos Storage configuration
VOLCENGINE_TOS_ENDPOINT=your-endpoint
VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name
VOLCENGINE_TOS_ACCESS_KEY=your-access-key
VOLCENGINE_TOS_SECRET_KEY=your-secret-key
VOLCENGINE_TOS_REGION=your-region
# CORS configuration # CORS configuration
WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,* WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,* CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*

View File

@ -12,6 +12,7 @@ from configs.middleware.storage.google_cloud_storage_config import GoogleCloudSt
from configs.middleware.storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig from configs.middleware.storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
from configs.middleware.storage.oci_storage_config import OCIStorageConfig from configs.middleware.storage.oci_storage_config import OCIStorageConfig
from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
from configs.middleware.storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
from configs.middleware.vdb.chroma_config import ChromaConfig from configs.middleware.vdb.chroma_config import ChromaConfig
from configs.middleware.vdb.elasticsearch_config import ElasticsearchConfig from configs.middleware.vdb.elasticsearch_config import ElasticsearchConfig
@ -201,6 +202,7 @@ class MiddlewareConfig(
GoogleCloudStorageConfig, GoogleCloudStorageConfig,
TencentCloudCOSStorageConfig, TencentCloudCOSStorageConfig,
HuaweiCloudOBSStorageConfig, HuaweiCloudOBSStorageConfig,
VolcengineTOSStorageConfig,
S3StorageConfig, S3StorageConfig,
OCIStorageConfig, OCIStorageConfig,
# configs of vdb and vdb providers # configs of vdb and vdb providers

View File

@ -0,0 +1,34 @@
from typing import Optional
from pydantic import BaseModel, Field
class VolcengineTOSStorageConfig(BaseModel):
"""
Volcengine tos storage configs
"""
VOLCENGINE_TOS_BUCKET_NAME: Optional[str] = Field(
description="Volcengine TOS Bucket Name",
default=None,
)
VOLCENGINE_TOS_ACCESS_KEY: Optional[str] = Field(
description="Volcengine TOS Access Key",
default=None,
)
VOLCENGINE_TOS_SECRET_KEY: Optional[str] = Field(
description="Volcengine TOS Secret Key",
default=None,
)
VOLCENGINE_TOS_ENDPOINT: Optional[str] = Field(
description="Volcengine TOS Endpoint URL",
default=None,
)
VOLCENGINE_TOS_REGION: Optional[str] = Field(
description="Volcengine TOS Region",
default=None,
)

View File

@ -11,6 +11,7 @@ from extensions.storage.local_storage import LocalStorage
from extensions.storage.oci_storage import OCIStorage from extensions.storage.oci_storage import OCIStorage
from extensions.storage.s3_storage import S3Storage from extensions.storage.s3_storage import S3Storage
from extensions.storage.tencent_storage import TencentStorage from extensions.storage.tencent_storage import TencentStorage
from extensions.storage.volcengine_storage import VolcengineStorage
class Storage: class Storage:
@ -33,6 +34,8 @@ class Storage:
self.storage_runner = OCIStorage(app=app) self.storage_runner = OCIStorage(app=app)
elif storage_type == "huawei-obs": elif storage_type == "huawei-obs":
self.storage_runner = HuaweiStorage(app=app) self.storage_runner = HuaweiStorage(app=app)
elif storage_type == "volcengine-tos":
self.storage_runner = VolcengineStorage(app=app)
else: else:
self.storage_runner = LocalStorage(app=app) self.storage_runner = LocalStorage(app=app)

View File

@ -0,0 +1,48 @@
from collections.abc import Generator
import tos
from flask import Flask
from extensions.storage.base_storage import BaseStorage
class VolcengineStorage(BaseStorage):
"""Implementation for Volcengine TOS storage."""
def __init__(self, app: Flask):
super().__init__(app)
app_config = self.app.config
self.bucket_name = app_config.get("VOLCENGINE_TOS_BUCKET_NAME")
self.client = tos.TosClientV2(
ak=app_config.get("VOLCENGINE_TOS_ACCESS_KEY"),
sk=app_config.get("VOLCENGINE_TOS_SECRET_KEY"),
endpoint=app_config.get("VOLCENGINE_TOS_ENDPOINT"),
region=app_config.get("VOLCENGINE_TOS_REGION"),
)
def save(self, filename, data):
self.client.put_object(bucket=self.bucket_name, key=filename, content=data)
def load_once(self, filename: str) -> bytes:
data = self.client.get_object(bucket=self.bucket_name, key=filename).read()
return data
def load_stream(self, filename: str) -> Generator:
def generate(filename: str = filename) -> Generator:
response = self.client.get_object(bucket=self.bucket_name, key=filename)
while chunk := response.read(4096):
yield chunk
return generate()
def download(self, filename, target_filepath):
self.client.get_object_to_file(bucket=self.bucket_name, key=filename, file_path=target_filepath)
def exists(self, filename):
res = self.client.head_object(bucket=self.bucket_name, key=filename)
if res.status_code != 200:
return False
return True
def delete(self, filename):
self.client.delete_object(bucket=self.bucket_name, key=filename)

19
api/poetry.lock generated
View File

@ -9036,6 +9036,23 @@ files = [
{file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
] ]
[[package]]
name = "tos"
version = "2.7.1"
description = "Volc TOS (Tinder Object Storage) SDK"
optional = false
python-versions = "*"
files = [
{file = "tos-2.7.1.tar.gz", hash = "sha256:4bccdbff3cfd63eb44648bb44862903708c4b3e790f0dd55c96305baaeece805"},
]
[package.dependencies]
crcmod = ">=1.7"
Deprecated = ">=1.2.13,<2.0.0"
pytz = "*"
requests = ">=2.19.1,<3.dev0"
six = "*"
[[package]] [[package]]
name = "tqdm" name = "tqdm"
version = "4.66.5" version = "4.66.5"
@ -10371,4 +10388,4 @@ cffi = ["cffi (>=1.11)"]
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.0"
python-versions = ">=3.10,<3.13" python-versions = ">=3.10,<3.13"
content-hash = "c3c637d643f4dcb3e35d0e7f2a3a4fbaf2a730512a4ca31adce5884c94f07f57" content-hash = "2dbff415c3c9ca95c8dcfb59fc088ce2c0d00037c44f386a34c87c98e1d8b942"

View File

@ -192,6 +192,7 @@ azure-ai-ml = "^1.19.0"
azure-ai-inference = "^1.0.0b3" azure-ai-inference = "^1.0.0b3"
volcengine-python-sdk = {extras = ["ark"], version = "^1.0.98"} volcengine-python-sdk = {extras = ["ark"], version = "^1.0.98"}
oci = "^2.133.0" oci = "^2.133.0"
tos = "^2.7.1"
[tool.poetry.group.indriect.dependencies] [tool.poetry.group.indriect.dependencies]
kaleido = "0.2.1" kaleido = "0.2.1"
rank-bm25 = "~0.2.2" rank-bm25 = "~0.2.2"

View File

@ -329,6 +329,18 @@ HUAWEI_OBS_ACCESS_KEY=your-access-key
# The server url of the HUAWEI OBS service. # The server url of the HUAWEI OBS service.
HUAWEI_OBS_SERVER=your-server-url HUAWEI_OBS_SERVER=your-server-url
# Volcengine TOS Configuration
# The name of the Volcengine TOS bucket to use for storing files.
VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name
# The secret key to use for authenticating with the Volcengine TOS service.
VOLCENGINE_TOS_SECRET_KEY=your-secret-key
# The access key to use for authenticating with the Volcengine TOS service.
VOLCENGINE_TOS_ACCESS_KEY=your-access-key
# The endpoint of the Volcengine TOS service.
VOLCENGINE_TOS_ENDPOINT=your-server-url
# The region of the Volcengine TOS service.
VOLCENGINE_TOS_REGION=your-region
# ------------------------------ # ------------------------------
# Vector Database Configuration # Vector Database Configuration
# ------------------------------ # ------------------------------

View File

@ -90,6 +90,11 @@ x-shared-env: &shared-api-worker-env
OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-} OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-}
OCI_SECRET_KEY: ${OCI_SECRET_KEY:-} OCI_SECRET_KEY: ${OCI_SECRET_KEY:-}
OCI_REGION: ${OCI_REGION:-} OCI_REGION: ${OCI_REGION:-}
VOLCENGINE_TOS_BUCKET_NAME: ${VOLCENGINE_TOS_BUCKET_NAME:-}
VOLCENGINE_TOS_SECRET_KEY: ${VOLCENGINE_TOS_SECRET_KEY:-}
VOLCENGINE_TOS_ACCESS_KEY: ${VOLCENGINE_TOS_ACCESS_KEY:-}
VOLCENGINE_TOS_ENDPOINT: ${VOLCENGINE_TOS_ENDPOINT:-}
VOLCENGINE_TOS_REGION: ${VOLCENGINE_TOS_REGION:-}
VECTOR_STORE: ${VECTOR_STORE:-weaviate} VECTOR_STORE: ${VECTOR_STORE:-weaviate}
WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080} WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080}
WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih}