KMS/dify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix text split

Browse Source
This commit is contained in:
jyong 2025-03-11 16:53:32 +08:00
parent e2e1a7add0
commit e90642d624
3 changed files with 41 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
api/core/rag/retrieval/dataset_retrieval.py
View File

@ -7,7 +7,7 @@ from collections.abc import Generator, Mapping
from typing import Any, Optional, Union, cast
from flask import Flask, current_app
from sqlalchemy import Integer, and_, or_
from sqlalchemy import Integer, and_, or_, text
from sqlalchemy import cast as sqlalchemy_cast
from core.app.app_config.entities import (
@ -847,7 +847,7 @@ class DatasetRetrieval:
for condition in metadata_filtering_conditions.conditions:
metadata_name = condition.name
expected_value = condition.value
if expected_value:
if expected_value or condition.comparison_operator in ("empty", "not empty"):
if isinstance(expected_value, str):
expected_value = self._replace_metadata_filter_value(expected_value, inputs)
filters = self._process_metadata_filter_func(
@ -930,17 +930,27 @@ class DatasetRetrieval:
return None
return automatic_metadata_filters
def _process_metadata_filter_func(self, condition: str, metadata_name: str, value: str, filters: list):
def _process_metadata_filter_func(self, condition: str, metadata_name: str, value: Optional[str], filters: list):
match condition:
case "contains":
filters.append(DatasetDocument.doc_metadata[metadata_name].like(f'"%{value}%"'))
filters.append(
(text("documents.doc_metadata ->> :key LIKE :value")).params(key=metadata_name, value=f"%{value}%")
)
case "not contains":
filters.append(DatasetDocument.doc_metadata[metadata_name].notlike(f'"%{value}%"'))
filters.append(
(text("documents.doc_metadata ->> :key NOT LIKE :value")).params(
key=metadata_name, value=f"%{value}%"
)
)
case "start with":
filters.append(DatasetDocument.doc_metadata[metadata_name].like(f'"{value}%"'))
filters.append(
(text("documents.doc_metadata ->> :key LIKE :value")).params(key=metadata_name, value=f"{value}%")
)
case "end with":
filters.append(DatasetDocument.doc_metadata[metadata_name].like(f'"%{value}"'))
filters.append(
(text("documents.doc_metadata ->> :key LIKE :value")).params(key=metadata_name, value=f"%{value}")
)
case "is" | "=":
if isinstance(value, str):
filters.append(DatasetDocument.doc_metadata[metadata_name] == f'"{value}"')
@ -955,9 +965,9 @@ class DatasetRetrieval:
filters.append(
sqlalchemy_cast(DatasetDocument.doc_metadata[metadata_name].astext, Integer) != value
)
case "is empty":
case "empty":
filters.append(DatasetDocument.doc_metadata[metadata_name].is_(None))
case "is not empty":
case "not empty":
filters.append(DatasetDocument.doc_metadata[metadata_name].isnot(None))
case "before" | "<":
filters.append(sqlalchemy_cast(DatasetDocument.doc_metadata[metadata_name].astext, Integer) < value)

32
api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py
View File

@ -4,7 +4,7 @@ from collections import defaultdict
from collections.abc import Mapping, Sequence
from typing import Any, Optional, cast
from sqlalchemy import and_, func, or_
from sqlalchemy import and_, func, or_, text
from core.app.app_config.entities import DatasetRetrieveConfigEntity
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
@ -293,13 +293,14 @@ class KnowledgeRetrievalNode(LLMNode):
Document.enabled == True,
Document.archived == False,
)
filters = []
metadata_condition = None
if node_data.metadata_filtering_mode == "disabled":
return None, None
elif node_data.metadata_filtering_mode == "automatic":
automatic_metadata_filters = self._automatic_metadata_filter_func(dataset_ids, query, node_data)
if automatic_metadata_filters:
conditions = []
filters = []
for filter in automatic_metadata_filters:
self._process_metadata_filter_func(
filter.get("condition"), filter.get("metadata_name"), filter.get("value"), filters
@ -318,10 +319,9 @@ class KnowledgeRetrievalNode(LLMNode):
elif node_data.metadata_filtering_mode == "manual":
if node_data.metadata_filtering_conditions:
for condition in node_data.metadata_filtering_conditions.conditions:
filters = []
metadata_name = condition.name
expected_value = condition.value
if expected_value:
if expected_value or condition.comparison_operator in ("empty", "not empty"):
if isinstance(expected_value, str):
expected_value = self.graph_runtime_state.variable_pool.convert_template(
expected_value
@ -407,16 +407,26 @@ class KnowledgeRetrievalNode(LLMNode):
return []
return automatic_metadata_filters
def _process_metadata_filter_func(self, condition: str, metadata_name: str, value: str, filters: list):
def _process_metadata_filter_func(self, condition: str, metadata_name: str, value: Optional[str], filters: list):
match condition:
case "contains":
filters.append(Document.doc_metadata[metadata_name].like(f'"%{value}%"'))
filters.append(
(text("documents.doc_metadata ->> :key LIKE :value")).params(key=metadata_name, value=f"%{value}%")
)
case "not contains":
filters.append(Document.doc_metadata[metadata_name].notlike(f'"%{value}%"'))
filters.append(
(text("documents.doc_metadata ->> :key NOT LIKE :value")).params(
key=metadata_name, value=f"%{value}%"
)
)
case "start with":
filters.append(Document.doc_metadata[metadata_name].like(f'"{value}%"'))
filters.append(
(text("documents.doc_metadata ->> :key LIKE :value")).params(key=metadata_name, value=f"{value}%")
)
case "end with":
filters.append(Document.doc_metadata[metadata_name].like(f'"%{value}"'))
filters.append(
(text("documents.doc_metadata ->> :key LIKE :value")).params(key=metadata_name, value=f"%{value}")
)
case "=" | "is":
if isinstance(value, str):
filters.append(Document.doc_metadata[metadata_name] == f'"{value}"')
@ -427,9 +437,9 @@ class KnowledgeRetrievalNode(LLMNode):
filters.append(Document.doc_metadata[metadata_name] != f'"{value}"')
else:
filters.append(Document.doc_metadata[metadata_name] != value)
case "is empty":
case "empty":
filters.append(Document.doc_metadata[metadata_name].is_(None))
case "is not empty":
case "not empty":
filters.append(Document.doc_metadata[metadata_name].isnot(None))
case "before" | "<":
filters.append(Document.doc_metadata[metadata_name] < value)