KMS/dify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dataset metadata fix

Browse Source
This commit is contained in:
jyong 2025-02-27 15:30:37 +08:00
parent c53786d229
commit 9e258c495d
21 changed files with 45 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/controllers/console/datasets/datasets_document.py
View File

@ -617,7 +617,7 @@ class DocumentDetailApi(DocumentResource):
raise InvalidMetadataError(f"Invalid metadata value: {metadata}")
if metadata == "only":
response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata}
response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
elif metadata == "without":
dataset_process_rules = DatasetService.get_process_rules(dataset_id)
document_process_rules = document.dataset_process_rule.to_dict()
@ -678,7 +678,7 @@ class DocumentDetailApi(DocumentResource):
"disabled_by": document.disabled_by,
"archived": document.archived,
"doc_type": document.doc_type,
"doc_metadata": document.doc_metadata,
"doc_metadata": document.doc_metadata_details,
"segment_count": document.segment_count,
"average_segment_length": document.average_segment_length,
"hit_count": document.hit_count,

8
api/core/rag/datasource/vdb/analyticdb/analyticdb_vector_sql.py
View File

@ -197,8 +197,8 @@ class AnalyticdbVectorBySql:
document_ids_filter = kwargs.get("document_ids_filter")
where_clause = "WHERE 1=1"
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause += f"AND metadata_->>'doc_id' IN ({doc_ids})"
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause += f"AND metadata_->>'document_id' IN ({document_ids})"
score_threshold = float(kwargs.get("score_threshold") or 0.0)
with self._get_cursor() as cur:
query_vector_str = json.dumps(query_vector)
@ -228,8 +228,8 @@ class AnalyticdbVectorBySql:
document_ids_filter = kwargs.get("document_ids_filter")
where_clause = ""
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause += f"AND metadata_->>'doc_id' IN ({doc_ids})"
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause += f"AND metadata_->>'document_id' IN ({document_ids})"
with self._get_cursor() as cur:
cur.execute(
f"""SELECT id, vector, page_content, metadata_,

4
api/core/rag/datasource/vdb/baidu/baidu_vector.py
View File

@ -125,12 +125,12 @@ class BaiduVector(BaseVector):
query_vector = [float(val) if isinstance(val, np.float64) else val for val in query_vector]
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
anns = AnnSearch(
vector_field=self.field_vector,
vector_floats=query_vector,
params=HNSWSearchParams(ef=kwargs.get("ef", 10), limit=kwargs.get("top_k", 4)),
filter=f"doc_id IN ({doc_ids})",
filter=f"document_id IN ({document_ids})",
)
else:
anns = AnnSearch(

2
api/core/rag/datasource/vdb/chroma/chroma_vector.py
View File

@ -100,7 +100,7 @@ class ChromaVector(BaseVector):
results: QueryResult = collection.query(
query_embeddings=query_vector,
n_results=kwargs.get("top_k", 4),
where={"doc_id": {"$in": document_ids_filter}},
where={"document_id": {"$in": document_ids_filter}},
)
else:
results: QueryResult = collection.query(query_embeddings=query_vector, n_results=kwargs.get("top_k", 4))

4
api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py
View File

@ -119,7 +119,7 @@ class ElasticSearchVector(BaseVector):
knn = {"field": Field.VECTOR.value, "query_vector": query_vector, "k": top_k, "num_candidates": num_candidates}
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
knn["filter"] = {"terms": {"metadata.doc_id": document_ids_filter}}
knn["filter"] = {"terms": {"metadata.document_id": document_ids_filter}}
results = self._client.search(index=self._collection_name, knn=knn, size=top_k)
@ -150,7 +150,7 @@ class ElasticSearchVector(BaseVector):
query_str = {"match": {Field.CONTENT_KEY.value: query}}
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
query_str["filter"] = {"terms": {"metadata.doc_id": document_ids_filter}}
query_str["filter"] = {"terms": {"metadata.document_id": document_ids_filter}}
results = self._client.search(index=self._collection_name, query=query_str, size=kwargs.get("top_k", 4))
docs = []
for hit in results["hits"]["hits"]:

4
api/core/rag/datasource/vdb/lindorm/lindorm_vector.py
View File

@ -171,7 +171,7 @@ class LindormVectorStore(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
filters = []
if document_ids_filter:
filters.append({"terms": {"metadata.doc_id": document_ids_filter}})
filters.append({"terms": {"metadata.document_id": document_ids_filter}})
query = default_vector_search_query(query_vector=query_vector, k=top_k, filters=filters, **kwargs)
try:
@ -214,7 +214,7 @@ class LindormVectorStore(BaseVector):
filters = kwargs.get("filter", [])
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
filters.append({"terms": {"metadata.doc_id": document_ids_filter}})
filters.append({"terms": {"metadata.document_id": document_ids_filter}})
routing = self._routing
full_text_query = default_text_search_query(
query_text=query,

8
api/core/rag/datasource/vdb/milvus/milvus_vector.py
View File

@ -221,8 +221,8 @@ class MilvusVector(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
filter = ""
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
filter = f'metadata["doc_id"] in ({doc_ids})'
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
filter = f'metadata["document_id"] in ({document_ids})'
results = self._client.search(
collection_name=self._collection_name,
data=[query_vector],
@ -248,8 +248,8 @@ class MilvusVector(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
filter = ""
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
filter = f'metadata["doc_id"] in ({doc_ids})'
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
filter = f'metadata["document_id"] in ({document_ids})'
results = self._client.search(
collection_name=self._collection_name,

4
api/core/rag/datasource/vdb/myscale/myscale_vector.py
View File

@ -133,8 +133,8 @@ class MyScaleVector(BaseVector):
)
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_str = f"{where_str} AND metadata['doc_id'] in ({doc_ids})"
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_str = f"{where_str} AND metadata['document_id'] in ({document_ids})"
sql = f"""
SELECT text, vector, metadata, {dist} as dist FROM {self._config.database}.{self._collection_name}
{where_str} ORDER BY dist {order.value} LIMIT {top_k}

4
api/core/rag/datasource/vdb/oceanbase/oceanbase_vector.py
View File

@ -157,8 +157,8 @@ class OceanBaseVector(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
where_clause = None
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f"metadata->>'$.doc_id' in ({doc_ids})"
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f"metadata->>'$.document_id' in ({document_ids})"
ef_search = kwargs.get("ef_search", self._hnsw_ef_search)
if ef_search != self._hnsw_ef_search:
self._client.set_ob_hnsw_ef_search(ef_search)

4
api/core/rag/datasource/vdb/opensearch/opensearch_vector.py
View File

@ -156,7 +156,7 @@ class OpenSearchVector(BaseVector):
}
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
query["query"] = {"terms": {"metadata.doc_id": document_ids_filter}}
query["query"] = {"terms": {"metadata.document_id": document_ids_filter}}
try:
response = self._client.search(index=self._collection_name.lower(), body=query)
@ -184,7 +184,7 @@ class OpenSearchVector(BaseVector):
full_text_query = {"query": {"match": {Field.CONTENT_KEY.value: query}}}
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
full_text_query["query"]["terms"] = {"metadata.doc_id": document_ids_filter}
full_text_query["query"]["terms"] = {"metadata.document_id": document_ids_filter}
response = self._client.search(index=self._collection_name.lower(), body=full_text_query)

8
api/core/rag/datasource/vdb/oracle/oraclevector.py
View File

@ -188,8 +188,8 @@ class OracleVector(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
where_clause = ""
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f"WHERE metadata->>'doc_id' in ({doc_ids})"
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f"WHERE metadata->>'document_id' in ({document_ids})"
with self._get_cursor() as cur:
cur.execute(
f"SELECT meta, text, vector_distance(embedding,:1) AS distance FROM {self.table_name}"
@ -249,8 +249,8 @@ class OracleVector(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
where_clause = ""
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f" AND metadata->>'doc_id' in ({doc_ids}) "
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f" AND metadata->>'document_id' in ({document_ids}) "
cur.execute(
f"select meta, text, embedding FROM {self.table_name}"
f"WHERE CONTAINS(text, :1, 1) > 0 {where_clause} "

2
api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py
View File

@ -191,7 +191,7 @@ class PGVectoRS(BaseVector):
)
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
stmt = stmt.where(self._table.meta["doc_id"].in_(document_ids_filter))
stmt = stmt.where(self._table.meta["document_id"].in_(document_ids_filter))
res = session.execute(stmt)
results = [(row[0], row[1]) for row in res]

8
api/core/rag/datasource/vdb/pgvector/pgvector.py
View File

@ -158,8 +158,8 @@ class PGVector(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
where_clause = ""
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f" WHERE metadata->>'doc_id' in ({doc_ids}) "
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f" WHERE metadata->>'document_id' in ({document_ids}) "
with self._get_cursor() as cur:
cur.execute(
@ -185,8 +185,8 @@ class PGVector(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
where_clause = ""
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f" AND metadata->>'doc_id' in ({doc_ids}) "
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f" AND metadata->>'document_id' in ({document_ids}) "
cur.execute(
f"""SELECT meta, text, ts_rank(to_tsvector(coalesce(text, '')), plainto_tsquery(%s)) AS score
FROM {self.table_name}

4
api/core/rag/datasource/vdb/qdrant/qdrant_vector.py
View File

@ -334,7 +334,7 @@ class QdrantVector(BaseVector):
if document_ids_filter:
filter.must.append(
models.FieldCondition(
key="metadata.doc_id",
key="metadata.document_id",
match=models.MatchAny(any=document_ids_filter),
)
)
@ -388,7 +388,7 @@ class QdrantVector(BaseVector):
if document_ids_filter:
scroll_filter.must.append(
models.FieldCondition(
key="metadata.doc_id",
key="metadata.document_id",
match=models.MatchAny(any=document_ids_filter),
)
)

2
api/core/rag/datasource/vdb/relyt/relyt_vector.py
View File

@ -226,7 +226,7 @@ class RelytVector(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
filter = kwargs.get("filter", {})
if document_ids_filter:
filter["doc_id"] = document_ids_filter
filter["document_id"] = document_ids_filter
results = self.similarity_search_with_score_by_vector(
k=int(kwargs.get("top_k", 4)), embedding=query_vector, filter=filter
)

2
api/core/rag/datasource/vdb/tencent/tencent_vector.py
View File

@ -151,7 +151,7 @@ class TencentVector(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
filter = None
if document_ids_filter:
filter = Filter(Filter.In("metadata.doc_id", document_ids_filter))
filter = Filter(Filter.In("metadata.document_id", document_ids_filter))
res = self._db.collection(self._collection_name).search(
vectors=[query_vector],
filter=filter,

4
api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py
View File

@ -330,7 +330,7 @@ class TidbOnQdrantVector(BaseVector):
if document_ids_filter:
filter.must.append(
models.FieldCondition(
key="metadata.doc_id",
key="metadata.document_id",
match=models.MatchAny(any=document_ids_filter),
)
)
@ -380,7 +380,7 @@ class TidbOnQdrantVector(BaseVector):
if document_ids_filter:
scroll_filter.must.append(
models.FieldCondition(
key="metadata.doc_id",
key="metadata.document_id",
match=models.MatchAny(any=document_ids_filter),
)
)

4
api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py
View File

@ -199,8 +199,8 @@ class TiDBVector(BaseVector):
document_ids_filter = kwargs.get("document_ids_filter")
where_clause = ""
if document_ids_filter:
doc_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f" WHERE meta->>'$.doc_id' in ({doc_ids}) "
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
where_clause = f" WHERE meta->>'$.document_id' in ({document_ids}) "
with Session(self._engine) as session:
select_statement = sql_text(f"""

3
api/core/rag/datasource/vdb/upstash/upstash_vector.py
View File

@ -90,7 +90,8 @@ class UpstashVector(BaseVector):
top_k = kwargs.get("top_k", 4)
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
filter = f"doc_id in ({', '.join(f"'{id}'" for id in document_ids_filter)})"
document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
filter = f"document_id in ({document_ids})"
else:
filter = ""
result = self.index.query(

2
api/core/rag/datasource/vdb/vikingdb/vikingdb_vector.py
View File

@ -180,7 +180,7 @@ class VikingDBVector(BaseVector):
docs = self._get_search_res(results, score_threshold)
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
docs = [doc for doc in docs if doc.metadata.get("doc_id") in document_ids_filter]
docs = [doc for doc in docs if doc.metadata.get("document_id") in document_ids_filter]
return docs
def _get_search_res(self, results, score_threshold) -> list[Document]:

4
api/core/rag/datasource/vdb/weaviate/weaviate_vector.py
View File

@ -189,7 +189,7 @@ class WeaviateVector(BaseVector):
vector = {"vector": query_vector}
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
where_filter = {"operator": "ContainsAny", "path": ["doc_id"], "valueTextArray": document_ids_filter}
where_filter = {"operator": "ContainsAny", "path": ["document_id"], "valueTextArray": document_ids_filter}
query_obj = query_obj.with_where(where_filter)
result = (
query_obj.with_near_vector(vector)
@ -237,7 +237,7 @@ class WeaviateVector(BaseVector):
query_obj = self._client.query.get(collection_name, properties)
document_ids_filter = kwargs.get("document_ids_filter")
if document_ids_filter:
where_filter = {"operator": "ContainsAny", "path": ["doc_id"], "valueTextArray": document_ids_filter}
where_filter = {"operator": "ContainsAny", "path": ["document_id"], "valueTextArray": document_ids_filter}
query_obj = query_obj.with_where(where_filter)
query_obj = query_obj.with_additional(["vector"])
properties = ["text"]