KMS/dify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix metadata

Browse Source
This commit is contained in:
jyong 2025-03-06 15:54:46 +08:00
parent 7f6c6b7791
commit 54c33ca476
3 changed files with 23 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/core/rag/retrieval/dataset_retrieval.py
View File

@ -899,7 +899,7 @@ class DatasetRetrieval:
case "≥", ">=": case "≥", ">=":
query = query.filter(Document.doc_metadata[metadata_name] >= value) query = query.filter(Document.doc_metadata[metadata_name] >= value)
case _: case _:
raise ValueError(f"Invalid condition: {condition}") pass
def _fetch_model_config( def _fetch_model_config(
self, tenant_id: str, model: ModelConfig self, tenant_id: str, model: ModelConfig

2
api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py
View File

@ -407,7 +407,7 @@ class KnowledgeRetrievalNode(LLMNode):
case "≥" | ">=": case "≥" | ">=":
query = query.filter(Document.doc_metadata[metadata_name] >= value) query = query.filter(Document.doc_metadata[metadata_name] >= value)
case _: case _:
raise InvalidConditionError(f"Invalid condition: {condition}") pass
@classmethod @classmethod
def _extract_variable_selector_to_variable_mapping( def _extract_variable_selector_to_variable_mapping(

35
api/services/metadata_service.py
View File

@ -1,3 +1,4 @@
import copy
import datetime import datetime
import logging import logging
from typing import Optional from typing import Optional
@ -30,7 +31,8 @@ class MetadataService:
return metadata return metadata
@staticmethod @staticmethod
def update_metadata_name(dataset_id: str, metadata_id: str, name: str) -> DatasetMetadata: def \
update_metadata_name(dataset_id: str, metadata_id: str, name: str) -> DatasetMetadata:
lock_key = f"dataset_metadata_lock_{dataset_id}" lock_key = f"dataset_metadata_lock_{dataset_id}"
try: try:
MetadataService.knowledge_base_metadata_lock_check(dataset_id, None) MetadataService.knowledge_base_metadata_lock_check(dataset_id, None)
@ -48,7 +50,10 @@ class MetadataService:
document_ids = [binding.document_id for binding in dataset_metadata_bindings] document_ids = [binding.document_id for binding in dataset_metadata_bindings]
documents = DocumentService.get_document_by_ids(document_ids) documents = DocumentService.get_document_by_ids(document_ids)
for document in documents: for document in documents:
document.doc_metadata[name] = document.doc_metadata.pop(old_name) doc_metadata = copy.deepcopy(document.doc_metadata)
value = doc_metadata.pop(old_name, None)
doc_metadata[name] = value
document.doc_metadata = doc_metadata
db.session.add(document) db.session.add(document)
db.session.commit() db.session.commit()
return metadata return metadata
@ -67,13 +72,15 @@ class MetadataService:
raise ValueError("Metadata not found.") raise ValueError("Metadata not found.")
db.session.delete(metadata) db.session.delete(metadata)
# delete related documents # deal related documents
dataset_metadata_bindings = DatasetMetadataBinding.query.filter_by(metadata_id=metadata_id).all() dataset_metadata_bindings = DatasetMetadataBinding.query.filter_by(metadata_id=metadata_id).all()
if dataset_metadata_bindings: if dataset_metadata_bindings:
document_ids = [binding.document_id for binding in dataset_metadata_bindings] document_ids = [binding.document_id for binding in dataset_metadata_bindings]
documents = DocumentService.get_document_by_ids(document_ids) documents = DocumentService.get_document_by_ids(document_ids)
for document in documents: for document in documents:
document.doc_metadata.pop(metadata.name) doc_metadata = copy.deepcopy(document.doc_metadata)
doc_metadata.pop(metadata.name, None)
document.doc_metadata = doc_metadata
db.session.add(document) db.session.add(document)
db.session.commit() db.session.commit()
return metadata return metadata
@ -87,8 +94,8 @@ class MetadataService:
return [ return [
{"name": BuiltInField.document_name, "type": "string"}, {"name": BuiltInField.document_name, "type": "string"},
{"name": BuiltInField.uploader, "type": "string"}, {"name": BuiltInField.uploader, "type": "string"},
{"name": BuiltInField.upload_date, "type": "date"}, {"name": BuiltInField.upload_date, "type": "time"},
{"name": BuiltInField.last_update_date, "type": "date"}, {"name": BuiltInField.last_update_date, "type": "time"},
{"name": BuiltInField.source, "type": "string"}, {"name": BuiltInField.source, "type": "string"},
] ]
@ -107,7 +114,7 @@ class MetadataService:
if not document.doc_metadata: if not document.doc_metadata:
doc_metadata = {} doc_metadata = {}
else: else:
doc_metadata = document.doc_metadata doc_metadata = copy.deepcopy(document.doc_metadata)
doc_metadata[BuiltInField.document_name.value] = document.name doc_metadata[BuiltInField.document_name.value] = document.name
doc_metadata[BuiltInField.uploader.value] = document.uploader doc_metadata[BuiltInField.uploader.value] = document.uploader
doc_metadata[BuiltInField.upload_date.value] = document.upload_date.timestamp() doc_metadata[BuiltInField.upload_date.value] = document.upload_date.timestamp()
@ -134,12 +141,12 @@ class MetadataService:
document_ids = [] document_ids = []
if documents: if documents:
for document in documents: for document in documents:
doc_metadata = document.doc_metadata doc_metadata = copy.deepcopy(document.doc_metadata)
doc_metadata.pop(BuiltInField.document_name) doc_metadata.pop(BuiltInField.document_name, None)
doc_metadata.pop(BuiltInField.uploader) doc_metadata.pop(BuiltInField.uploader, None)
doc_metadata.pop(BuiltInField.upload_date) doc_metadata.pop(BuiltInField.upload_date, None)
doc_metadata.pop(BuiltInField.last_update_date) doc_metadata.pop(BuiltInField.last_update_date, None)
doc_metadata.pop(BuiltInField.source) doc_metadata.pop(BuiltInField.source, None)
document.doc_metadata = doc_metadata document.doc_metadata = doc_metadata
db.session.add(document) db.session.add(document)
document_ids.append(document.id) document_ids.append(document.id)
@ -170,7 +177,7 @@ class MetadataService:
document.doc_metadata = doc_metadata document.doc_metadata = doc_metadata
db.session.add(document) db.session.add(document)
db.session.commit() db.session.commit()
# deal metadata bindding # deal metadata binding
DatasetMetadataBinding.query.filter_by(document_id=operation.document_id).delete() DatasetMetadataBinding.query.filter_by(document_id=operation.document_id).delete()
for metadata_value in operation.metadata_list: for metadata_value in operation.metadata_list:
dataset_metadata_binding = DatasetMetadataBinding( dataset_metadata_binding = DatasetMetadataBinding(