mirror of
https://github.com/langgenius/dify.git
synced 2024-11-16 03:32:23 +08:00
add volcengine tos storage (#8164)
This commit is contained in:
parent
144d30d7ef
commit
c8df92d0eb
|
@ -39,7 +39,7 @@ DB_DATABASE=dify
|
||||||
|
|
||||||
# Storage configuration
|
# Storage configuration
|
||||||
# use for store upload files, private keys...
|
# use for store upload files, private keys...
|
||||||
# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs
|
# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs, volcengine-tos
|
||||||
STORAGE_TYPE=local
|
STORAGE_TYPE=local
|
||||||
STORAGE_LOCAL_PATH=storage
|
STORAGE_LOCAL_PATH=storage
|
||||||
S3_USE_AWS_MANAGED_IAM=false
|
S3_USE_AWS_MANAGED_IAM=false
|
||||||
|
@ -86,6 +86,13 @@ OCI_ACCESS_KEY=your-access-key
|
||||||
OCI_SECRET_KEY=your-secret-key
|
OCI_SECRET_KEY=your-secret-key
|
||||||
OCI_REGION=your-region
|
OCI_REGION=your-region
|
||||||
|
|
||||||
|
# Volcengine tos Storage configuration
|
||||||
|
VOLCENGINE_TOS_ENDPOINT=your-endpoint
|
||||||
|
VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name
|
||||||
|
VOLCENGINE_TOS_ACCESS_KEY=your-access-key
|
||||||
|
VOLCENGINE_TOS_SECRET_KEY=your-secret-key
|
||||||
|
VOLCENGINE_TOS_REGION=your-region
|
||||||
|
|
||||||
# CORS configuration
|
# CORS configuration
|
||||||
WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
|
WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
|
||||||
CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
|
CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
|
||||||
|
|
|
@ -12,6 +12,7 @@ from configs.middleware.storage.google_cloud_storage_config import GoogleCloudSt
|
||||||
from configs.middleware.storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
|
from configs.middleware.storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
|
||||||
from configs.middleware.storage.oci_storage_config import OCIStorageConfig
|
from configs.middleware.storage.oci_storage_config import OCIStorageConfig
|
||||||
from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
|
from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
|
||||||
|
from configs.middleware.storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
|
||||||
from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
|
from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
|
||||||
from configs.middleware.vdb.chroma_config import ChromaConfig
|
from configs.middleware.vdb.chroma_config import ChromaConfig
|
||||||
from configs.middleware.vdb.elasticsearch_config import ElasticsearchConfig
|
from configs.middleware.vdb.elasticsearch_config import ElasticsearchConfig
|
||||||
|
@ -201,6 +202,7 @@ class MiddlewareConfig(
|
||||||
GoogleCloudStorageConfig,
|
GoogleCloudStorageConfig,
|
||||||
TencentCloudCOSStorageConfig,
|
TencentCloudCOSStorageConfig,
|
||||||
HuaweiCloudOBSStorageConfig,
|
HuaweiCloudOBSStorageConfig,
|
||||||
|
VolcengineTOSStorageConfig,
|
||||||
S3StorageConfig,
|
S3StorageConfig,
|
||||||
OCIStorageConfig,
|
OCIStorageConfig,
|
||||||
# configs of vdb and vdb providers
|
# configs of vdb and vdb providers
|
||||||
|
|
|
@ -0,0 +1,34 @@
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
|
||||||
|
class VolcengineTOSStorageConfig(BaseModel):
|
||||||
|
"""
|
||||||
|
Volcengine tos storage configs
|
||||||
|
"""
|
||||||
|
|
||||||
|
VOLCENGINE_TOS_BUCKET_NAME: Optional[str] = Field(
|
||||||
|
description="Volcengine TOS Bucket Name",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
VOLCENGINE_TOS_ACCESS_KEY: Optional[str] = Field(
|
||||||
|
description="Volcengine TOS Access Key",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
VOLCENGINE_TOS_SECRET_KEY: Optional[str] = Field(
|
||||||
|
description="Volcengine TOS Secret Key",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
VOLCENGINE_TOS_ENDPOINT: Optional[str] = Field(
|
||||||
|
description="Volcengine TOS Endpoint URL",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
VOLCENGINE_TOS_REGION: Optional[str] = Field(
|
||||||
|
description="Volcengine TOS Region",
|
||||||
|
default=None,
|
||||||
|
)
|
|
@ -11,6 +11,7 @@ from extensions.storage.local_storage import LocalStorage
|
||||||
from extensions.storage.oci_storage import OCIStorage
|
from extensions.storage.oci_storage import OCIStorage
|
||||||
from extensions.storage.s3_storage import S3Storage
|
from extensions.storage.s3_storage import S3Storage
|
||||||
from extensions.storage.tencent_storage import TencentStorage
|
from extensions.storage.tencent_storage import TencentStorage
|
||||||
|
from extensions.storage.volcengine_storage import VolcengineStorage
|
||||||
|
|
||||||
|
|
||||||
class Storage:
|
class Storage:
|
||||||
|
@ -33,6 +34,8 @@ class Storage:
|
||||||
self.storage_runner = OCIStorage(app=app)
|
self.storage_runner = OCIStorage(app=app)
|
||||||
elif storage_type == "huawei-obs":
|
elif storage_type == "huawei-obs":
|
||||||
self.storage_runner = HuaweiStorage(app=app)
|
self.storage_runner = HuaweiStorage(app=app)
|
||||||
|
elif storage_type == "volcengine-tos":
|
||||||
|
self.storage_runner = VolcengineStorage(app=app)
|
||||||
else:
|
else:
|
||||||
self.storage_runner = LocalStorage(app=app)
|
self.storage_runner = LocalStorage(app=app)
|
||||||
|
|
||||||
|
|
48
api/extensions/storage/volcengine_storage.py
Normal file
48
api/extensions/storage/volcengine_storage.py
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
from collections.abc import Generator
|
||||||
|
|
||||||
|
import tos
|
||||||
|
from flask import Flask
|
||||||
|
|
||||||
|
from extensions.storage.base_storage import BaseStorage
|
||||||
|
|
||||||
|
|
||||||
|
class VolcengineStorage(BaseStorage):
|
||||||
|
"""Implementation for Volcengine TOS storage."""
|
||||||
|
|
||||||
|
def __init__(self, app: Flask):
|
||||||
|
super().__init__(app)
|
||||||
|
app_config = self.app.config
|
||||||
|
self.bucket_name = app_config.get("VOLCENGINE_TOS_BUCKET_NAME")
|
||||||
|
self.client = tos.TosClientV2(
|
||||||
|
ak=app_config.get("VOLCENGINE_TOS_ACCESS_KEY"),
|
||||||
|
sk=app_config.get("VOLCENGINE_TOS_SECRET_KEY"),
|
||||||
|
endpoint=app_config.get("VOLCENGINE_TOS_ENDPOINT"),
|
||||||
|
region=app_config.get("VOLCENGINE_TOS_REGION"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def save(self, filename, data):
|
||||||
|
self.client.put_object(bucket=self.bucket_name, key=filename, content=data)
|
||||||
|
|
||||||
|
def load_once(self, filename: str) -> bytes:
|
||||||
|
data = self.client.get_object(bucket=self.bucket_name, key=filename).read()
|
||||||
|
return data
|
||||||
|
|
||||||
|
def load_stream(self, filename: str) -> Generator:
|
||||||
|
def generate(filename: str = filename) -> Generator:
|
||||||
|
response = self.client.get_object(bucket=self.bucket_name, key=filename)
|
||||||
|
while chunk := response.read(4096):
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
return generate()
|
||||||
|
|
||||||
|
def download(self, filename, target_filepath):
|
||||||
|
self.client.get_object_to_file(bucket=self.bucket_name, key=filename, file_path=target_filepath)
|
||||||
|
|
||||||
|
def exists(self, filename):
|
||||||
|
res = self.client.head_object(bucket=self.bucket_name, key=filename)
|
||||||
|
if res.status_code != 200:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def delete(self, filename):
|
||||||
|
self.client.delete_object(bucket=self.bucket_name, key=filename)
|
19
api/poetry.lock
generated
19
api/poetry.lock
generated
|
@ -9036,6 +9036,23 @@ files = [
|
||||||
{file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
|
{file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tos"
|
||||||
|
version = "2.7.1"
|
||||||
|
description = "Volc TOS (Tinder Object Storage) SDK"
|
||||||
|
optional = false
|
||||||
|
python-versions = "*"
|
||||||
|
files = [
|
||||||
|
{file = "tos-2.7.1.tar.gz", hash = "sha256:4bccdbff3cfd63eb44648bb44862903708c4b3e790f0dd55c96305baaeece805"},
|
||||||
|
]
|
||||||
|
|
||||||
|
[package.dependencies]
|
||||||
|
crcmod = ">=1.7"
|
||||||
|
Deprecated = ">=1.2.13,<2.0.0"
|
||||||
|
pytz = "*"
|
||||||
|
requests = ">=2.19.1,<3.dev0"
|
||||||
|
six = "*"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tqdm"
|
name = "tqdm"
|
||||||
version = "4.66.5"
|
version = "4.66.5"
|
||||||
|
@ -10371,4 +10388,4 @@ cffi = ["cffi (>=1.11)"]
|
||||||
[metadata]
|
[metadata]
|
||||||
lock-version = "2.0"
|
lock-version = "2.0"
|
||||||
python-versions = ">=3.10,<3.13"
|
python-versions = ">=3.10,<3.13"
|
||||||
content-hash = "c3c637d643f4dcb3e35d0e7f2a3a4fbaf2a730512a4ca31adce5884c94f07f57"
|
content-hash = "2dbff415c3c9ca95c8dcfb59fc088ce2c0d00037c44f386a34c87c98e1d8b942"
|
||||||
|
|
|
@ -192,6 +192,7 @@ azure-ai-ml = "^1.19.0"
|
||||||
azure-ai-inference = "^1.0.0b3"
|
azure-ai-inference = "^1.0.0b3"
|
||||||
volcengine-python-sdk = {extras = ["ark"], version = "^1.0.98"}
|
volcengine-python-sdk = {extras = ["ark"], version = "^1.0.98"}
|
||||||
oci = "^2.133.0"
|
oci = "^2.133.0"
|
||||||
|
tos = "^2.7.1"
|
||||||
[tool.poetry.group.indriect.dependencies]
|
[tool.poetry.group.indriect.dependencies]
|
||||||
kaleido = "0.2.1"
|
kaleido = "0.2.1"
|
||||||
rank-bm25 = "~0.2.2"
|
rank-bm25 = "~0.2.2"
|
||||||
|
|
|
@ -329,6 +329,18 @@ HUAWEI_OBS_ACCESS_KEY=your-access-key
|
||||||
# The server url of the HUAWEI OBS service.
|
# The server url of the HUAWEI OBS service.
|
||||||
HUAWEI_OBS_SERVER=your-server-url
|
HUAWEI_OBS_SERVER=your-server-url
|
||||||
|
|
||||||
|
# Volcengine TOS Configuration
|
||||||
|
# The name of the Volcengine TOS bucket to use for storing files.
|
||||||
|
VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name
|
||||||
|
# The secret key to use for authenticating with the Volcengine TOS service.
|
||||||
|
VOLCENGINE_TOS_SECRET_KEY=your-secret-key
|
||||||
|
# The access key to use for authenticating with the Volcengine TOS service.
|
||||||
|
VOLCENGINE_TOS_ACCESS_KEY=your-access-key
|
||||||
|
# The endpoint of the Volcengine TOS service.
|
||||||
|
VOLCENGINE_TOS_ENDPOINT=your-server-url
|
||||||
|
# The region of the Volcengine TOS service.
|
||||||
|
VOLCENGINE_TOS_REGION=your-region
|
||||||
|
|
||||||
# ------------------------------
|
# ------------------------------
|
||||||
# Vector Database Configuration
|
# Vector Database Configuration
|
||||||
# ------------------------------
|
# ------------------------------
|
||||||
|
|
|
@ -90,6 +90,11 @@ x-shared-env: &shared-api-worker-env
|
||||||
OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-}
|
OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-}
|
||||||
OCI_SECRET_KEY: ${OCI_SECRET_KEY:-}
|
OCI_SECRET_KEY: ${OCI_SECRET_KEY:-}
|
||||||
OCI_REGION: ${OCI_REGION:-}
|
OCI_REGION: ${OCI_REGION:-}
|
||||||
|
VOLCENGINE_TOS_BUCKET_NAME: ${VOLCENGINE_TOS_BUCKET_NAME:-}
|
||||||
|
VOLCENGINE_TOS_SECRET_KEY: ${VOLCENGINE_TOS_SECRET_KEY:-}
|
||||||
|
VOLCENGINE_TOS_ACCESS_KEY: ${VOLCENGINE_TOS_ACCESS_KEY:-}
|
||||||
|
VOLCENGINE_TOS_ENDPOINT: ${VOLCENGINE_TOS_ENDPOINT:-}
|
||||||
|
VOLCENGINE_TOS_REGION: ${VOLCENGINE_TOS_REGION:-}
|
||||||
VECTOR_STORE: ${VECTOR_STORE:-weaviate}
|
VECTOR_STORE: ${VECTOR_STORE:-weaviate}
|
||||||
WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080}
|
WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080}
|
||||||
WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih}
|
WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user