From f6c7204002ca876a40f97c09ed0eed57e08e8bbf Mon Sep 17 00:00:00 2001 From: KevinHuSh Date: Thu, 11 Apr 2024 10:13:43 +0800 Subject: [PATCH] refine log format (#312) ### What problem does this PR solve? Issue link:#264 ### Type of change - [x] Documentation Update - [x] Refactoring --- README.md | 2 +- README_ja.md | 2 +- README_zh.md | 2 +- api/db/db_models.py | 2 +- api/settings.py | 16 ++++++---------- api/utils/log_utils.py | 3 ++- rag/app/book.py | 2 +- rag/app/laws.py | 2 +- rag/llm/chat_model.py | 2 +- 9 files changed, 15 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5000d5e6..686f6bb4 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ - CPU >= 2 cores - RAM >= 8 GB -- Docker >= 24.0.0 +- Docker >= 24.0.0 & Docker Compose >= v2.26.1 > If you have not installed Docker on your local machine (Windows, Mac, or Linux), see [Install Docker Engine](https://docs.docker.com/engine/install/). ### 🚀 Start up the server diff --git a/README_ja.md b/README_ja.md index cf198e7f..d3074bd1 100644 --- a/README_ja.md +++ b/README_ja.md @@ -65,7 +65,7 @@ - CPU >= 2 cores - RAM >= 8 GB -- Docker >= 24.0.0 +- Docker >= 24.0.0 & Docker Compose >= v2.26.1 > ローカルマシン（Windows、Mac、または Linux）に Docker をインストールしていない場合は、[Docker Engine のインストール](https://docs.docker.com/engine/install/) を参照してください。 ### 🚀 サーバーを起動 diff --git a/README_zh.md b/README_zh.md index 2479b4b5..7c7d571b 100644 --- a/README_zh.md +++ b/README_zh.md @@ -65,7 +65,7 @@ - CPU >= 2 核 - RAM >= 8 GB -- Docker >= 24.0.0 +- Docker >= 24.0.0 & Docker Compose >= v2.26.1 > 如果你并没有在本机安装 Docker（Windows、Mac，或者 Linux）, 可以参考文档 [Install Docker Engine](https://docs.docker.com/engine/install/) 自行安装。 ### 🚀 启动服务器 diff --git a/api/db/db_models.py b/api/db/db_models.py index b66badf6..6d7adf0d 100644 --- a/api/db/db_models.py +++ b/api/db/db_models.py @@ -695,7 +695,7 @@ class Dialog(DataBaseModel): language = CharField( max_length=32, null=True, - default="English", + default="Chinese", help_text="English|Chinese") llm_id = CharField(max_length=32, null=False, help_text="default llm ID") llm_setting = JSONField(null=False, default={"temperature": 0.1, "top_p": 0.3, "frequency_penalty": 0.7, diff --git a/api/settings.py b/api/settings.py index be4ae85a..6010ab19 100644 --- a/api/settings.py +++ b/api/settings.py @@ -13,15 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import logging - -from rag.utils import ELASTICSEARCH -from rag.nlp import search import os - from enum import IntEnum, Enum - -from api.utils import get_base_config, decrypt_database_config from api.utils.file_utils import get_project_base_directory from api.utils.log_utils import LoggerFactory, getLogger @@ -32,13 +25,16 @@ LoggerFactory.set_directory( "logs", "api")) # {CRITICAL: 50, FATAL:50, ERROR:40, WARNING:30, WARN:30, INFO:20, DEBUG:10, NOTSET:0} -LoggerFactory.LEVEL = 10 +LoggerFactory.LEVEL = 30 stat_logger = getLogger("stat") access_logger = getLogger("access") database_logger = getLogger("database") chat_logger = getLogger("chat") -database_logger.setLevel(logging.WARNING) + +from rag.utils import ELASTICSEARCH +from rag.nlp import search +from api.utils import get_base_config, decrypt_database_config API_VERSION = "v1" RAG_FLOW_SERVICE_NAME = "ragflow" @@ -79,7 +75,7 @@ default_llm = { "image2text_model": "glm-4v", "asr_model": "", }, - "Local": { + "Ollama": { "chat_model": "qwen-14B-chat", "embedding_model": "flag-embedding", "image2text_model": "", diff --git a/api/utils/log_utils.py b/api/utils/log_utils.py index 528bd99a..e74060d4 100644 --- a/api/utils/log_utils.py +++ b/api/utils/log_utils.py @@ -26,7 +26,8 @@ from api.utils import file_utils class LoggerFactory(object): TYPE = "FILE" - LOG_FORMAT = "[%(levelname)s] [%(asctime)s] [jobId] [%(process)s:%(thread)s] - [%(module)s.%(funcName)s] [line:%(lineno)d]: %(message)s" + LOG_FORMAT = "[%(levelname)s] [%(asctime)s] [%(module)s.%(funcName)s] [line:%(lineno)d]: %(message)s" + logging.basicConfig(format=LOG_FORMAT) LEVEL = logging.DEBUG logger_dict = {} global_handler_dict = {} diff --git a/rag/app/book.py b/rag/app/book.py index 2ea850af..85661aca 100644 --- a/rag/app/book.py +++ b/rag/app/book.py @@ -110,7 +110,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, [t for t in random_choices([t for t, _ in sections], k=100)]) if bull >= 0: chunks = ["\n".join(ck) - for ck in hierarchical_merge(bull, sections, 3)] + for ck in hierarchical_merge(bull, sections, 5)] else: sections = [s.split("@") for s, _ in sections] sections = [(pr[0], "@" + pr[1]) for pr in sections if len(pr) == 2] diff --git a/rag/app/laws.py b/rag/app/laws.py index ce0d1747..2a7842f7 100644 --- a/rag/app/laws.py +++ b/rag/app/laws.py @@ -133,7 +133,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, make_colon_as_title(sections) bull = bullets_category(sections) - chunks = hierarchical_merge(bull, sections, 3) + chunks = hierarchical_merge(bull, sections, 5) if not chunks: callback(0.99, "No chunk parsed out.") diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.py index 498595de..d4f0e7b6 100644 --- a/rag/llm/chat_model.py +++ b/rag/llm/chat_model.py @@ -98,7 +98,7 @@ class QWenChat(Base): tk_count = 0 if response.status_code == HTTPStatus.OK: ans += response.output.choices[0]['message']['content'] - tk_count += response.usage.output_tokens + tk_count += response.usage.total_tokens if response.output.choices[0].get("finish_reason", "") == "length": ans += "...\nFor the content length reason, it stopped, continue?" if is_english( [ans]) else "······\n由于长度的原因，回答被截断了，要继续吗？"