diff --git a/api/.env.example b/api/.env.example index 822166c68b..573c8bf90c 100644 --- a/api/.env.example +++ b/api/.env.example @@ -72,6 +72,13 @@ TENCENT_COS_SECRET_ID=your-secret-id TENCENT_COS_REGION=your-region TENCENT_COS_SCHEME=your-scheme +# OCI Storage configuration +OCI_ENDPOINT=your-endpoint +OCI_BUCKET_NAME=your-bucket-name +OCI_ACCESS_KEY=your-access-key +OCI_SECRET_KEY=your-secret-key +OCI_REGION=your-region + # CORS configuration WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,* CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,* diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index b6f962e3d5..b1957efb6b 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -7,6 +7,7 @@ from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorag from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig +from configs.middleware.storage.oci_storage_config import OCIStorageConfig from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig from configs.middleware.vdb.chroma_config import ChromaConfig from configs.middleware.vdb.milvus_config import MilvusConfig @@ -167,6 +168,7 @@ class MiddlewareConfig( GoogleCloudStorageConfig, TencentCloudCOSStorageConfig, S3StorageConfig, + OCIStorageConfig, # configs of vdb and vdb providers VectorStoreConfig, diff --git a/api/configs/middleware/storage/oci_storage_config.py b/api/configs/middleware/storage/oci_storage_config.py new file mode 100644 index 0000000000..5fd99c6d1e --- /dev/null +++ b/api/configs/middleware/storage/oci_storage_config.py @@ -0,0 +1,35 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class OCIStorageConfig(BaseModel): + """ + OCI storage configs + """ + + OCI_ENDPOINT: Optional[str] = Field( + description='OCI storage endpoint', + default=None, + ) + + OCI_REGION: Optional[str] = Field( + description='OCI storage region', + default=None, + ) + + OCI_BUCKET_NAME: Optional[str] = Field( + description='OCI storage bucket name', + default=None, + ) + + OCI_ACCESS_KEY: Optional[str] = Field( + description='OCI storage access key', + default=None, + ) + + OCI_SECRET_KEY: Optional[str] = Field( + description='OCI storage secret key', + default=None, + ) + diff --git a/api/extensions/ext_storage.py b/api/extensions/ext_storage.py index 130f2ea69d..38db1c6ce1 100644 --- a/api/extensions/ext_storage.py +++ b/api/extensions/ext_storage.py @@ -7,6 +7,7 @@ from extensions.storage.aliyun_storage import AliyunStorage from extensions.storage.azure_storage import AzureStorage from extensions.storage.google_storage import GoogleStorage from extensions.storage.local_storage import LocalStorage +from extensions.storage.oci_storage import OCIStorage from extensions.storage.s3_storage import S3Storage from extensions.storage.tencent_storage import TencentStorage @@ -37,6 +38,10 @@ class Storage: self.storage_runner = TencentStorage( app=app ) + elif storage_type == 'oci-storage': + self.storage_runner = OCIStorage( + app=app + ) else: self.storage_runner = LocalStorage(app=app) diff --git a/api/extensions/storage/oci_storage.py b/api/extensions/storage/oci_storage.py new file mode 100644 index 0000000000..e78d870950 --- /dev/null +++ b/api/extensions/storage/oci_storage.py @@ -0,0 +1,64 @@ +from collections.abc import Generator +from contextlib import closing + +import boto3 +from botocore.exceptions import ClientError +from flask import Flask + +from extensions.storage.base_storage import BaseStorage + + +class OCIStorage(BaseStorage): + def __init__(self, app: Flask): + super().__init__(app) + app_config = self.app.config + self.bucket_name = app_config.get('OCI_BUCKET_NAME') + self.client = boto3.client( + 's3', + aws_secret_access_key=app_config.get('OCI_SECRET_KEY'), + aws_access_key_id=app_config.get('OCI_ACCESS_KEY'), + endpoint_url=app_config.get('OCI_ENDPOINT'), + region_name=app_config.get('OCI_REGION') + ) + + def save(self, filename, data): + self.client.put_object(Bucket=self.bucket_name, Key=filename, Body=data) + + def load_once(self, filename: str) -> bytes: + try: + with closing(self.client) as client: + data = client.get_object(Bucket=self.bucket_name, Key=filename)['Body'].read() + except ClientError as ex: + if ex.response['Error']['Code'] == 'NoSuchKey': + raise FileNotFoundError("File not found") + else: + raise + return data + + def load_stream(self, filename: str) -> Generator: + def generate(filename: str = filename) -> Generator: + try: + with closing(self.client) as client: + response = client.get_object(Bucket=self.bucket_name, Key=filename) + yield from response['Body'].iter_chunks() + except ClientError as ex: + if ex.response['Error']['Code'] == 'NoSuchKey': + raise FileNotFoundError("File not found") + else: + raise + return generate() + + def download(self, filename, target_filepath): + with closing(self.client) as client: + client.download_file(self.bucket_name, filename, target_filepath) + + def exists(self, filename): + with closing(self.client) as client: + try: + client.head_object(Bucket=self.bucket_name, Key=filename) + return True + except: + return False + + def delete(self, filename): + self.client.delete_object(Bucket=self.bucket_name, Key=filename) \ No newline at end of file diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 0846666bc5..f1a1ffcf56 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -64,6 +64,11 @@ x-shared-env: &shared-api-worker-env TENCENT_COS_SECRET_ID: ${TENCENT_COS_SECRET_ID:-} TENCENT_COS_REGION: ${TENCENT_COS_REGION:-} TENCENT_COS_SCHEME: ${TENCENT_COS_SCHEME:-} + OCI_ENDPOINT: ${OCI_ENDPOINT:-} + OCI_BUCKET_NAME: ${OCI_BUCKET_NAME:-} + OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-} + OCI_SECRET_KEY: ${OCI_SECRET_KEY:-} + OCI_REGION: ${OCI_REGION:-} VECTOR_STORE: ${VECTOR_STORE:-weaviate} WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080} WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih}