diff --git a/api/.env b/api/.env new file mode 100644 index 0000000000..0a3705192e --- /dev/null +++ b/api/.env @@ -0,0 +1,170 @@ +# Server Edition +EDITION=SELF_HOSTED + +# Your App secret key will be used for securely signing the session cookie +# Make sure you are changing this key for your deployment with a strong key. +# You can generate a strong key using `openssl rand -base64 42`. +# Alternatively you can set it with `SECRET_KEY` environment variable. +SECRET_KEY= + +# Console API base URL +CONSOLE_API_URL=http://127.0.0.1:5001 +CONSOLE_WEB_URL=http://127.0.0.1:3000 + +# Service API base URL +SERVICE_API_URL=http://127.0.0.1:5001 + +# Web APP base URL +APP_WEB_URL=http://127.0.0.1:3000 + +# Files URL +FILES_URL=http://127.0.0.1:5001 + +# celery configuration +CELERY_BROKER_URL=redis://:difyai123456@localhost:6379/1 + +# redis configuration +REDIS_HOST=localhost +REDIS_PORT=6379 +REDIS_USERNAME= +REDIS_PASSWORD=difyai123456 +REDIS_DB=0 + +# PostgreSQL database configuration +DB_USERNAME=postgres +DB_PASSWORD=difyai123456 +DB_HOST=localhost +DB_PORT=5432 +DB_DATABASE=dify + +# Storage configuration +# use for store upload files, private keys... +# storage type: local, s3, azure-blob +STORAGE_TYPE=local +STORAGE_LOCAL_PATH=storage +S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com +S3_BUCKET_NAME=your-bucket-name +S3_ACCESS_KEY=your-access-key +S3_SECRET_KEY=your-secret-key +S3_REGION=your-region +# Azure Blob Storage configuration +AZURE_BLOB_ACCOUNT_NAME=your-account-name +AZURE_BLOB_ACCOUNT_KEY=your-account-key +AZURE_BLOB_CONTAINER_NAME=yout-container-name +AZURE_BLOB_ACCOUNT_URL=https://.blob.core.windows.net + +# CORS configuration +WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,* +CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,* + +# Vector database configuration, support: weaviate, qdrant, milvus, relyt +VECTOR_STORE=tencent + +TENCENT_URL=http://10.6.1.224 +TENCENT_API_KEY=nTZEVu0UeShVmMXkMywZQpMLC3BCERM7nLOPH2Xf +TENCENT_TIMEOUT=30 +TENCENT_USERNAME=root +TENCENT_DATABASE=dify +TENCENT_SHARD=1 +TENCENT_REPLICAS=2 + +# Weaviate configuration +WEAVIATE_ENDPOINT=http://localhost:8080 +WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih +WEAVIATE_GRPC_ENABLED=false +WEAVIATE_BATCH_SIZE=100 + +# Qdrant configuration, use `http://localhost:6333` for local mode or `https://your-qdrant-cluster-url.qdrant.io` for remote mode +QDRANT_URL=http://localhost:6333 +QDRANT_API_KEY=difyai123456 +QDRANT_CLIENT_TIMEOUT=20 + +# Milvus configuration +MILVUS_HOST=127.0.0.1 +MILVUS_PORT=19530 +MILVUS_USER=root +MILVUS_PASSWORD=Milvus +MILVUS_SECURE=false + +# Relyt configuration +RELYT_HOST=127.0.0.1 +RELYT_PORT=5432 +RELYT_USER=postgres +RELYT_PASSWORD=postgres +RELYT_DATABASE=postgres + +# Upload configuration +UPLOAD_FILE_SIZE_LIMIT=15 +UPLOAD_FILE_BATCH_LIMIT=5 +UPLOAD_IMAGE_FILE_SIZE_LIMIT=10 + +# Model Configuration +MULTIMODAL_SEND_IMAGE_FORMAT=base64 + +# Mail configuration, support: resend, smtp +MAIL_TYPE= +MAIL_DEFAULT_SEND_FROM=no-reply +RESEND_API_KEY= +RESEND_API_URL=https://api.resend.com +# smtp configuration +SMTP_SERVER=smtp.gmail.com +SMTP_PORT=587 +SMTP_USERNAME=123 +SMTP_PASSWORD=abc +SMTP_USE_TLS=false + +# Sentry configuration +SENTRY_DSN= + +# DEBUG +DEBUG=false +SQLALCHEMY_ECHO=false + +# Notion import configuration, support public and internal +NOTION_INTEGRATION_TYPE=public +NOTION_CLIENT_SECRET=you-client-secret +NOTION_CLIENT_ID=you-client-id +NOTION_INTERNAL_SECRET=you-internal-secret + +# Hosted Model Credentials +HOSTED_OPENAI_API_KEY= +HOSTED_OPENAI_API_BASE= +HOSTED_OPENAI_API_ORGANIZATION= +HOSTED_OPENAI_TRIAL_ENABLED=false +HOSTED_OPENAI_QUOTA_LIMIT=200 +HOSTED_OPENAI_PAID_ENABLED=false + +HOSTED_AZURE_OPENAI_ENABLED=false +HOSTED_AZURE_OPENAI_API_KEY= +HOSTED_AZURE_OPENAI_API_BASE= +HOSTED_AZURE_OPENAI_QUOTA_LIMIT=200 + +HOSTED_ANTHROPIC_API_BASE= +HOSTED_ANTHROPIC_API_KEY= +HOSTED_ANTHROPIC_TRIAL_ENABLED=false +HOSTED_ANTHROPIC_QUOTA_LIMIT=600000 +HOSTED_ANTHROPIC_PAID_ENABLED=false + +ETL_TYPE=dify +UNSTRUCTURED_API_URL= + +SSRF_PROXY_HTTP_URL= +SSRF_PROXY_HTTPS_URL= + +BATCH_UPLOAD_LIMIT=10 +KEYWORD_DATA_SOURCE_TYPE=database + +# CODE EXECUTION CONFIGURATION +CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194 +CODE_EXECUTION_API_KEY=dify-sandbox +CODE_MAX_NUMBER=9223372036854775807 +CODE_MIN_NUMBER=-9223372036854775808 +CODE_MAX_STRING_LENGTH=80000 +TEMPLATE_TRANSFORM_MAX_LENGTH=80000 +CODE_MAX_STRING_ARRAY_LENGTH=30 +CODE_MAX_OBJECT_ARRAY_LENGTH=30 +CODE_MAX_NUMBER_ARRAY_LENGTH=1000 + +# API Tool configuration +API_TOOL_DEFAULT_CONNECT_TIMEOUT=10 +API_TOOL_DEFAULT_READ_TIMEOUT=60 diff --git a/api/.env.example b/api/.env.example index 481d6ab499..24f7829f54 100644 --- a/api/.env.example +++ b/api/.env.example @@ -85,6 +85,15 @@ RELYT_USER=postgres RELYT_PASSWORD=postgres RELYT_DATABASE=postgres +# Tencent configuration +TENCENT_URL=http://127.0.0.1 +TENCENT_API_KEY=dify +TENCENT_TIMEOUT=30 +TENCENT_USERNAME=dify +TENCENT_DATABASE=dify +TENCENT_SHARD=1 +TENCENT_REPLICAS=2 + # Upload configuration UPLOAD_FILE_SIZE_LIMIT=15 UPLOAD_FILE_BATCH_LIMIT=5 diff --git a/api/commands.py b/api/commands.py index b82f7ac3f8..9212d10faa 100644 --- a/api/commands.py +++ b/api/commands.py @@ -305,6 +305,14 @@ def migrate_knowledge_vector_database(): "vector_store": {"class_prefix": collection_name} } dataset.index_struct = json.dumps(index_struct_dict) + elif vector_type == "tencent": + dataset_id = dataset.id + collection_name = Dataset.gen_collection_name_by_id(dataset_id) + index_struct_dict = { + "type": 'tencent', + "vector_store": {"class_prefix": collection_name} + } + dataset.index_struct = json.dumps(index_struct_dict) else: raise ValueError(f"Vector store {config.get('VECTOR_STORE')} is not supported.") diff --git a/api/config.py b/api/config.py index f210ac48f9..261c54952f 100644 --- a/api/config.py +++ b/api/config.py @@ -228,6 +228,15 @@ class Config: self.RELYT_PASSWORD = get_env('RELYT_PASSWORD') self.RELYT_DATABASE = get_env('RELYT_DATABASE') + # tencent settings + self.TENCENT_URL = get_env('TENCENT_URL') + self.TENCENT_API_KEY = get_env('TENCENT_API_KEY') + self.TENCENT_TIMEOUT = get_env('TENCENT_TIMEOUT') + self.TENCENT_USERNAME = get_env('TENCENT_USERNAME') + self.TENCENT_DATABASE = get_env('TENCENT_DATABASE') + self.TENCENT_SHARD = get_env('TENCENT_SHARD') + self.TENCENT_REPLICAS = get_env('TENCENT_REPLICAS') + # ------------------------ # Mail Configurations. # ------------------------ diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index e633631c42..d656baa135 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -475,7 +475,7 @@ class DatasetRetrievalSettingApi(Resource): 'semantic_search' ] } - elif vector_type == 'qdrant' or vector_type == 'weaviate': + elif vector_type == 'qdrant' or vector_type == 'weaviate' or vector_type == 'tencent': return { 'retrieval_method': [ 'semantic_search', 'full_text_search', 'hybrid_search' @@ -497,7 +497,7 @@ class DatasetRetrievalSettingMockApi(Resource): 'semantic_search' ] } - elif vector_type == 'qdrant' or vector_type == 'weaviate': + elif vector_type == 'qdrant' or vector_type == 'weaviate' or vector_type == 'tencent': return { 'retrieval_method': [ 'semantic_search', 'full_text_search', 'hybrid_search' diff --git a/api/core/rag/datasource/vdb/tencent/__init__.py b/api/core/rag/datasource/vdb/tencent/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/rag/datasource/vdb/tencent/tencent_vector.py b/api/core/rag/datasource/vdb/tencent/tencent_vector.py new file mode 100644 index 0000000000..a2826fe9a9 --- /dev/null +++ b/api/core/rag/datasource/vdb/tencent/tencent_vector.py @@ -0,0 +1,182 @@ +import json +from typing import Any, Optional + +import tcvectordb +from pydantic import BaseModel +from tcvectordb.model import document, enum +from tcvectordb.model import index as vdb_index +from tcvectordb.model.document import Filter + +from core.rag.datasource.vdb.vector_base import BaseVector +from core.rag.models.document import Document +from extensions.ext_redis import redis_client + + +class TencentConfig(BaseModel): + url: str + api_key: Optional[str] + timeout: float = 30 + username: Optional[str] + database: Optional[str] + index_type: str = "HNSW" + metric_type: str = "L2" + shard: int = 1, + replicas: int = 2, + + def to_tencent_params(self): + return { + 'url': self.url, + 'username': self.username, + 'key': self.api_key, + 'timeout': self.timeout + } + + +class TencentVector(BaseVector): + field_id: str = "id" + field_vector: str = "vector" + field_text: str = "text" + field_metadata: str = "metadata" + + def __init__(self, collection_name: str, config: TencentConfig): + super().__init__(collection_name) + self._client_config = config + self._client = tcvectordb.VectorDBClient(**self._client_config.to_tencent_params()) + self._db = self._init_database() + + def _init_database(self): + exists = False + for db in self._client.list_databases(): + if db.database_name == self._client_config.database: + exists = True + break + if exists: + return self._client.database(self._client_config.database) + else: + return self._client.create_database(database_name=self._client_config.database) + + def get_type(self) -> str: + return 'tencent' + + def to_index_struct(self) -> dict: + return { + "type": self.get_type(), + "vector_store": {"class_prefix": self._collection_name} + } + + def _create_collection(self, dimension: int) -> None: + lock_name = 'vector_indexing_lock_{}'.format(self._collection_name) + with redis_client.lock(lock_name, timeout=20): + self.delete() + index_type = None + for k, v in enum.IndexType.__members__.items(): + if k == self._client_config.index_type: + index_type = v + if index_type is None: + raise ValueError("unsupported index_type") + metric_type = None + for k, v in enum.MetricType.__members__.items(): + if k == self._client_config.metric_type: + metric_type = v + if metric_type is None: + raise ValueError("unsupported metric_type") + params = vdb_index.HNSWParams(m=16, efconstruction=200) + index = vdb_index.Index( + vdb_index.FilterIndex( + self.field_id, enum.FieldType.String, enum.IndexType.PRIMARY_KEY + ), + vdb_index.VectorIndex( + self.field_vector, + dimension, + index_type, + metric_type, + params, + ), + vdb_index.FilterIndex( + self.field_text, enum.FieldType.String, enum.IndexType.FILTER + ), + vdb_index.FilterIndex( + self.field_metadata, enum.FieldType.String, enum.IndexType.FILTER + ), + ) + + self.collection = self._db.create_collection( + name=self._collection_name, + shard=self._client_config.shard, + replicas=self._client_config.replicas, + description="Collection for Dify", + index=index, + ) + + def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs): + self._create_collection(len(embeddings[0])) + self.add_texts(texts, embeddings) + + def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs): + texts = [doc.page_content for doc in documents] + metadatas = [doc.metadata for doc in documents] + total_count = len(embeddings) + docs = [] + for id in range(0, total_count): + if metadatas is None: + continue + metadata = json.dumps(metadatas[id]) + doc = document.Document( + id=metadatas[id]["doc_id"], + vector=embeddings[id], + text=texts[id], + metadata=metadata, + ) + docs.append(doc) + self.collection.upsert(docs, self._client_config.timeout) + + def text_exists(self, id: str) -> bool: + docs = self._db.collection(self._collection_name).query(document_ids=[id]) + if docs and len(docs) > 0: + return True + return False + + def delete_by_ids(self, ids: list[str]) -> None: + self._db.collection(self._collection_name).delete(document_ids=ids) + + def delete_by_metadata_field(self, key: str, value: str) -> None: + docs = self._db.collection(self._collection_name).query(filter=Filter(Filter.In(key, [value]))) + if docs and len(docs) > 0: + self.collection.delete(document_ids=[doc['id'] for doc in docs]) + + def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: + + res = self._db.collection(self._collection_name).search(vectors=[query_vector], + params=document.HNSWSearchParams( + ef=kwargs.get("ef", 10)), + retrieve_vector=False, + limit=kwargs.get('top_k', 4), + timeout=self._client_config.timeout, + ) + return self._get_search_res(res) + + def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: + res = (self._db.collection(self._collection_name) + .searchByText(embeddingItems=[query], + params=document.HNSWSearchParams(ef=kwargs.get("ef", 10)), + retrieve_vector=False, + limit=kwargs.get('top_k', 4), + timeout=self._client_config.timeout, + )) + return self._get_search_res(res) + + def _get_search_res(self, res): + docs = [] + if res is None or len(res) == 0: + return docs + + for result in res[0]: + meta = result.get(self.field_metadata) + if meta is not None: + meta = json.loads(meta) + doc = Document(page_content=result.get(self.field_text), metadata=meta) + docs.append(doc) + return docs + + def delete(self) -> None: + self._db.drop_collection(name=self._collection_name) diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index b6ec7a11fb..1d0d79064e 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -25,7 +25,6 @@ class Vector: def _init_vector(self) -> BaseVector: config = current_app.config vector_type = config.get('VECTOR_STORE') - if self._dataset.index_struct_dict: vector_type = self._dataset.index_struct_dict['type'] @@ -138,6 +137,31 @@ class Vector: ), dim=dim ) + elif vector_type == "tencent": + from core.rag.datasource.vdb.tencent.tencent_vector import TencentConfig, TencentVector + if self._dataset.index_struct_dict: + class_prefix: str = self._dataset.index_struct_dict['vector_store']['class_prefix'] + collection_name = class_prefix + else: + dataset_id = self._dataset.id + collection_name = Dataset.gen_collection_name_by_id(dataset_id) + index_struct_dict = { + "type": 'tencent', + "vector_store": {"class_prefix": collection_name} + } + self._dataset.index_struct = json.dumps(index_struct_dict) + return TencentVector( + collection_name=collection_name, + config=TencentConfig( + url=config.get('TENCENT_URL'), + api_key=config.get('TENCENT_API_KEY'), + timeout=config.get('TENCENT_TIMEOUT'), + username=config.get('TENCENT_USERNAME'), + database=config.get('TENCENT_DATABASE'), + shard=config.get('TENCENT_SHARD'), + replicas=config.get('TENCENT_REPLICAS'), + ) + ) else: raise ValueError(f"Vector store {config.get('VECTOR_STORE')} is not supported.") diff --git a/api/tests/unit_tests/core/rag/datasource/vdb/tencent/__init__.py b/api/tests/unit_tests/core/rag/datasource/vdb/tencent/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/core/rag/datasource/vdb/tencent/test_tencent.py b/api/tests/unit_tests/core/rag/datasource/vdb/tencent/test_tencent.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index dccdaee2cc..b894ecce2e 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -229,6 +229,14 @@ services: RELYT_USER: postgres RELYT_PASSWORD: difyai123456 RELYT_DATABASE: postgres + # tencent configurations + TENCENT_URL: http://127.0.0.1 + TENCENT_API_KEY: dify + TENCENT_TIMEOUT: 30 + TENCENT_USERNAME: dify + TENCENT_DATABASE: dify + TENCENT_SHARD: 1 + TENCENT_REPLICAS: 2 depends_on: - db - redis