From fe8f87834daa37c53b32e867cdc2bcb1cfe0630d Mon Sep 17 00:00:00 2001 From: CaptainB Date: Wed, 25 Jun 2025 16:46:38 +0800 Subject: [PATCH] refactor: replace logging with maxkb_logger for consistent logging across modules --- .../ai_chat_step_node/impl/base_chat_node.py | 3 +- apps/common/event/listener_manage.py | 33 +++++++++---------- .../impl/table/csv_parse_table_handle.py | 7 ++-- .../impl/table/xls_parse_table_handle.py | 5 ++- .../impl/table/xlsx_parse_table_handle.py | 7 ++-- .../handle/impl/text/pdf_split_handle.py | 10 +++--- apps/common/job/clean_chat_job.py | 5 +-- apps/common/job/clean_debug_file_job.py | 5 +-- apps/common/utils/fork.py | 12 ++++--- apps/knowledge/serializers/knowledge.py | 3 +- apps/knowledge/task/embedding.py | 15 ++++----- apps/knowledge/task/generate.py | 10 +++--- apps/knowledge/task/handler.py | 6 ++-- apps/knowledge/task/sync.py | 16 ++++----- .../model/tti.py | 20 ++++------- .../impl/xf_model_provider/model/stt.py | 1 - .../impl/xf_model_provider/model/tts.py | 1 - apps/ops/celery/logger.py | 5 +-- .../serializers/email_setting.py | 3 +- 19 files changed, 79 insertions(+), 88 deletions(-) diff --git a/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py b/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py index 8e1b48a7d..66556b40f 100644 --- a/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py +++ b/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py @@ -25,6 +25,7 @@ from application.flow.step_node.ai_chat_step_node.i_chat_node import IChatNode from application.flow.tools import Reasoning from models_provider.models import Model from models_provider.tools import get_model_credential, get_model_instance_by_model_workspace_id +from common.utils.logger import maxkb_logger tool_message_template = """
@@ -126,7 +127,7 @@ def mcp_response_generator(chat_model, message_list, mcp_servers): except StopAsyncIteration: break except Exception as e: - logging.getLogger("max_kb").error(f'Exception: {e}') + maxkb_logger.error(f'Exception: {e}') finally: loop.close() diff --git a/apps/common/event/listener_manage.py b/apps/common/event/listener_manage.py index 9c52e00e7..53305dc93 100644 --- a/apps/common/event/listener_manage.py +++ b/apps/common/event/listener_manage.py @@ -6,7 +6,6 @@ @date：2023/10/20 14:01 @desc: """ -import logging import os import threading import datetime @@ -22,13 +21,13 @@ from common.config.embedding_config import VectorStore from common.db.search import native_search, get_dynamics_model, native_update from common.utils.common import get_file_content from common.utils.lock import try_lock, un_lock +from common.utils.logger import maxkb_logger, maxkb_error_logger from common.utils.page_utils import page_desc from knowledge.models import Paragraph, Status, Document, ProblemParagraphMapping, TaskType, State,SourceType, SearchMode from maxkb.conf import (PROJECT_DIR) from django.utils.translation import gettext_lazy as _ -max_kb_error = logging.getLogger(__file__) -max_kb = logging.getLogger(__file__) + lock = threading.Lock() @@ -87,12 +86,12 @@ class ListenerManagement: ListenerManagement.embedding_by_paragraph_data_list(data_list, paragraph_id_list=paragraph_id_list, embedding_model=embedding_model) except Exception as e: - max_kb_error.error(_('Query vector data: {paragraph_id_list} error {error} {traceback}').format( + maxkb_error_logger.error(_('Query vector data: {paragraph_id_list} error {error} {traceback}').format( paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc())) @staticmethod def embedding_by_paragraph_data_list(data_list, paragraph_id_list, embedding_model: Embeddings): - max_kb.info(_('Start--->Embedding paragraph: {paragraph_id_list}').format(paragraph_id_list=paragraph_id_list)) + maxkb_logger.info(_('Start--->Embedding paragraph: {paragraph_id_list}').format(paragraph_id_list=paragraph_id_list)) status = Status.success try: # 删除段落 @@ -104,12 +103,12 @@ class ListenerManagement: # 批量向量化 VectorStore.get_embedding_vector().batch_save(data_list, embedding_model, is_save_function) except Exception as e: - max_kb_error.error(_('Vectorized paragraph: {paragraph_id_list} error {error} {traceback}').format( + maxkb_error_logger.error(_('Vectorized paragraph: {paragraph_id_list} error {error} {traceback}').format( paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc())) status = Status.error finally: QuerySet(Paragraph).filter(id__in=paragraph_id_list).update(**{'status': status}) - max_kb.info( + maxkb_logger.info( _('End--->Embedding paragraph: {paragraph_id_list}').format(paragraph_id_list=paragraph_id_list)) @staticmethod @@ -119,7 +118,7 @@ class ListenerManagement: @param paragraph_id: 段落id @param embedding_model: 向量模型 """ - max_kb.info(_('Start--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id)) + maxkb_logger.info(_('Start--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id)) # 更新到开始状态 ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING, State.STARTED) try: @@ -144,12 +143,12 @@ class ListenerManagement: ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING, State.SUCCESS) except Exception as e: - max_kb_error.error(_('Vectorized paragraph: {paragraph_id} error {error} {traceback}').format( + maxkb_error_logger.error(_('Vectorized paragraph: {paragraph_id} error {error} {traceback}').format( paragraph_id=paragraph_id, error=str(e), traceback=traceback.format_exc())) ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING, State.FAILURE) finally: - max_kb.info(_('End--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id)) + maxkb_logger.info(_('End--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id)) @staticmethod def embedding_by_data_list(data_list: List, embedding_model: Embeddings): @@ -265,7 +264,7 @@ class ListenerManagement: if is_the_task_interrupted(): return - max_kb.info(_('Start--->Embedding document: {document_id}').format(document_id=document_id) + maxkb_logger.info(_('Start--->Embedding document: {document_id}').format(document_id=document_id) ) # 批量修改状态为PADDING ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING, @@ -285,12 +284,12 @@ class ListenerManagement: document_id)), is_the_task_interrupted) except Exception as e: - max_kb_error.error(_('Vectorized document: {document_id} error {error} {traceback}').format( + maxkb_error_logger.error(_('Vectorized document: {document_id} error {error} {traceback}').format( document_id=document_id, error=str(e), traceback=traceback.format_exc())) finally: ListenerManagement.post_update_document_status(document_id, TaskType.EMBEDDING) ListenerManagement.get_aggregation_document_status(document_id)() - max_kb.info(_('End--->Embedding document: {document_id}').format(document_id=document_id)) + maxkb_logger.info(_('End--->Embedding document: {document_id}').format(document_id=document_id)) un_lock('embedding' + str(document_id)) @staticmethod @@ -301,18 +300,18 @@ class ListenerManagement: @param embedding_model 向量模型 :return: None """ - max_kb.info(_('Start--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) + maxkb_logger.info(_('Start--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) try: ListenerManagement.delete_embedding_by_knowledge(knowledge_id) document_list = QuerySet(Document).filter(knowledge_id=knowledge_id) - max_kb.info(_('Start--->Embedding document: {document_list}').format(document_list=document_list)) + maxkb_logger.info(_('Start--->Embedding document: {document_list}').format(document_list=document_list)) for document in document_list: ListenerManagement.embedding_by_document(document.id, embedding_model=embedding_model) except Exception as e: - max_kb_error.error(_('Vectorized knowledge: {knowledge_id} error {error} {traceback}').format( + maxkb_error_logger.error(_('Vectorized knowledge: {knowledge_id} error {error} {traceback}').format( knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc())) finally: - max_kb.info(_('End--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) + maxkb_logger.info(_('End--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) @staticmethod def delete_embedding_by_document(document_id): diff --git a/apps/common/handle/impl/table/csv_parse_table_handle.py b/apps/common/handle/impl/table/csv_parse_table_handle.py index ece767ff8..22370cc74 100644 --- a/apps/common/handle/impl/table/csv_parse_table_handle.py +++ b/apps/common/handle/impl/table/csv_parse_table_handle.py @@ -4,8 +4,7 @@ import logging from charset_normalizer import detect from common.handle.base_parse_table_handle import BaseParseTableHandle - -max_kb = logging.getLogger("max_kb") +from common.utils.logger import maxkb_logger, maxkb_error_logger class CsvParseTableHandle(BaseParseTableHandle): @@ -20,7 +19,7 @@ class CsvParseTableHandle(BaseParseTableHandle): try: content = buffer.decode(detect(buffer)['encoding']) except BaseException as e: - max_kb.error(f'csv split handle error: {e}') + maxkb_logger.error(f'csv split handle error: {e}') return [{'name': file.name, 'paragraphs': []}] csv_model = content.split('\n') @@ -40,5 +39,5 @@ class CsvParseTableHandle(BaseParseTableHandle): try: return buffer.decode(detect(buffer)['encoding']) except BaseException as e: - max_kb.error(f'csv split handle error: {e}') + maxkb_error_logger.error(f'csv split handle error: {e}') return f'error: {e}' diff --git a/apps/common/handle/impl/table/xls_parse_table_handle.py b/apps/common/handle/impl/table/xls_parse_table_handle.py index 8dcdf6030..5827e340c 100644 --- a/apps/common/handle/impl/table/xls_parse_table_handle.py +++ b/apps/common/handle/impl/table/xls_parse_table_handle.py @@ -4,8 +4,7 @@ import logging import xlrd from common.handle.base_parse_table_handle import BaseParseTableHandle - -max_kb = logging.getLogger("max_kb") +from common.utils.logger import maxkb_error_logger class XlsParseTableHandle(BaseParseTableHandle): @@ -56,7 +55,7 @@ class XlsParseTableHandle(BaseParseTableHandle): result.append({'name': sheet.name, 'paragraphs': paragraphs}) except BaseException as e: - max_kb.error(f'excel split handle error: {e}') + maxkb_error_logger.error(f'excel split handle error: {e}') return [{'name': file.name, 'paragraphs': []}] return result diff --git a/apps/common/handle/impl/table/xlsx_parse_table_handle.py b/apps/common/handle/impl/table/xlsx_parse_table_handle.py index 5bd7b47c0..e637022ea 100644 --- a/apps/common/handle/impl/table/xlsx_parse_table_handle.py +++ b/apps/common/handle/impl/table/xlsx_parse_table_handle.py @@ -6,8 +6,7 @@ from openpyxl import load_workbook from common.handle.base_parse_table_handle import BaseParseTableHandle from common.handle.impl.common_handle import xlsx_embed_cells_images - -max_kb = logging.getLogger("max_kb") +from common.utils.logger import maxkb_error_logger class XlsxParseTableHandle(BaseParseTableHandle): @@ -74,7 +73,7 @@ class XlsxParseTableHandle(BaseParseTableHandle): result.append({'name': sheetname, 'paragraphs': paragraphs}) except BaseException as e: - max_kb.error(f'excel split handle error: {e}') + maxkb_error_logger.error(f'excel split handle error: {e}') return [{'name': file.name, 'paragraphs': []}] return result @@ -87,7 +86,7 @@ class XlsxParseTableHandle(BaseParseTableHandle): if len(image_dict) > 0: save_image(image_dict.values()) except Exception as e: - logging.getLogger("max_kb").error(f'Exception: {e}') + maxkb_error_logger.error(f'Exception: {e}') image_dict = {} md_tables = '' # 如果未指定 sheet_name，则使用第一个工作表 diff --git a/apps/common/handle/impl/text/pdf_split_handle.py b/apps/common/handle/impl/text/pdf_split_handle.py index 0dd706912..18703fe9c 100644 --- a/apps/common/handle/impl/text/pdf_split_handle.py +++ b/apps/common/handle/impl/text/pdf_split_handle.py @@ -19,6 +19,7 @@ from django.utils.translation import gettext_lazy as _ from langchain_community.document_loaders import PyPDFLoader from common.handle.base_split_handle import BaseSplitHandle +from common.utils.logger import maxkb_error_logger, maxkb_logger from common.utils.split_model import SplitModel default_pattern_list = [re.compile('(?<=^)# .*|(?<=\\n)# .*'), @@ -29,7 +30,6 @@ default_pattern_list = [re.compile('(?<=^)# .*|(?<=\\n)# .*'), re.compile("(?<=\\n)(? 0: split_model = SplitModel(pattern_list, with_filter, limit) else: diff --git a/apps/common/job/clean_chat_job.py b/apps/common/job/clean_chat_job.py index 3c8ef5a8b..d5947adb8 100644 --- a/apps/common/job/clean_chat_job.py +++ b/apps/common/job/clean_chat_job.py @@ -10,6 +10,7 @@ from django_apscheduler.jobstores import DjangoJobStore from application.models import Application, Chat, ChatRecord from django.db.models import Q, Max from common.lock.impl.file_lock import FileLock +from common.utils.logger import maxkb_logger from knowledge.models import File @@ -20,7 +21,7 @@ lock = FileLock() def clean_chat_log_job(): from django.utils.translation import gettext_lazy as _ - logging.getLogger("max_kb").info(_('start clean chat log')) + maxkb_logger.info(_('start clean chat log')) now = timezone.now() applications = Application.objects.all().values('id', 'clean_time') @@ -66,7 +67,7 @@ def clean_chat_log_job(): if deleted_count < batch_size: break - logging.getLogger("max_kb").info(_('end clean chat log')) + maxkb_logger.info(_('end clean chat log')) def run(): diff --git a/apps/common/job/clean_debug_file_job.py b/apps/common/job/clean_debug_file_job.py index 30c61e1f5..19bccae4f 100644 --- a/apps/common/job/clean_debug_file_job.py +++ b/apps/common/job/clean_debug_file_job.py @@ -9,6 +9,7 @@ from django.utils import timezone from django_apscheduler.jobstores import DjangoJobStore from common.lock.impl.file_lock import FileLock +from common.utils.logger import maxkb_logger from knowledge.models import File scheduler = BackgroundScheduler() @@ -18,11 +19,11 @@ lock = FileLock() def clean_debug_file(): from django.utils.translation import gettext_lazy as _ - logging.getLogger("max_kb").info(_('start clean debug file')) + maxkb_logger.info(_('start clean debug file')) two_hours_ago = timezone.now() - timedelta(hours=2) # 删除对应的文件 File.objects.filter(Q(create_time__lt=two_hours_ago) & Q(meta__debug=True)).delete() - logging.getLogger("max_kb").info(_('end clean debug file')) + maxkb_logger.info(_('end clean debug file')) def run(): diff --git a/apps/common/utils/fork.py b/apps/common/utils/fork.py index 0672e4f90..c1e95e948 100644 --- a/apps/common/utils/fork.py +++ b/apps/common/utils/fork.py @@ -10,6 +10,8 @@ import html2text as ht import requests from bs4 import BeautifulSoup +from common.utils.logger import maxkb_error_logger, maxkb_logger + requests.packages.urllib3.disable_warnings() @@ -145,7 +147,7 @@ class Fork: try: html_content = response.content.decode(charset) except Exception as e: - logging.getLogger("max_kb").error(f'{e}') + maxkb_error_logger.error(f'{e}') return BeautifulSoup(html_content, "html.parser") return beautiful_soup @@ -156,14 +158,14 @@ class Fork: 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36' } - logging.getLogger("max_kb").info(f'fork:{self.base_fork_url}') + maxkb_logger.info(f'fork:{self.base_fork_url}') response = requests.get(self.base_fork_url, verify=False, headers=headers) if response.status_code != 200: - logging.getLogger("max_kb").error(f"url: {self.base_fork_url} code:{response.status_code}") + maxkb_error_logger.error(f"url: {self.base_fork_url} code:{response.status_code}") return Fork.Response.error(f"url: {self.base_fork_url} code:{response.status_code}") bf = self.get_beautiful_soup(response) except Exception as e: - logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}') + maxkb_error_logger.error(f'{str(e)}:{traceback.format_exc()}') return Fork.Response.error(str(e)) bf = self.reset_beautiful_soup(bf) link_list = self.get_child_link_list(bf) @@ -173,6 +175,6 @@ class Fork: def handler(base_url, response: Fork.Response): - logging.getLogger("max_kb").info(base_url.url, base_url.tag.text if base_url.tag else None, response.content) + maxkb_logger.info(base_url.url, base_url.tag.text if base_url.tag else None, response.content) # ForkManage('https://bbs.fit2cloud.com/c/de/6', ['.md-content']).fork(3, set(), handler) diff --git a/apps/knowledge/serializers/knowledge.py b/apps/knowledge/serializers/knowledge.py index a32bde5d8..e4762ef11 100644 --- a/apps/knowledge/serializers/knowledge.py +++ b/apps/knowledge/serializers/knowledge.py @@ -30,6 +30,7 @@ from common.event import ListenerManagement from common.exception.app_exception import AppApiException from common.utils.common import valid_license, post, get_file_content, parse_image from common.utils.fork import Fork, ChildLink +from common.utils.logger import maxkb_logger from common.utils.split_model import get_split_model from knowledge.models import Knowledge, KnowledgeScope, KnowledgeType, Document, Paragraph, Problem, \ ProblemParagraphMapping, TaskType, State, SearchMode, KnowledgeFolder @@ -630,7 +631,7 @@ class KnowledgeSerializer(serializers.Serializer): document_name = child_link.tag.text if child_link.tag is not None and len( child_link.tag.text.strip()) > 0 else child_link.url paragraphs = get_split_model('web.md').parse(response.content) - logging.getLogger("max_kb").info(child_link.url.strip()) + maxkb_logger.info(child_link.url.strip()) first = QuerySet(Document).filter( meta__source_url=child_link.url.strip(), knowledge=knowledge diff --git a/apps/knowledge/task/embedding.py b/apps/knowledge/task/embedding.py index 448c5091b..4f0f77335 100644 --- a/apps/knowledge/task/embedding.py +++ b/apps/knowledge/task/embedding.py @@ -11,16 +11,15 @@ from django.utils.translation import gettext_lazy as _ from common.config.embedding_config import ModelManage from common.event import ListenerManagement, UpdateProblemArgs, UpdateEmbeddingKnowledgeIdArgs, \ UpdateEmbeddingDocumentIdArgs +from common.utils.logger import maxkb_error_logger, maxkb_logger from knowledge.models import Document, TaskType, State from models_provider.tools import get_model from models_provider.models import Model from ops import celery_app -max_kb_error = logging.getLogger("max_kb_error") -max_kb = logging.getLogger("max_kb") -def get_embedding_model(model_id, exception_handler=lambda e: max_kb_error.error( +def get_embedding_model(model_id, exception_handler=lambda e: maxkb_error_logger.error( _('Failed to obtain vector model: {error} {traceback}').format( error=str(e), traceback=traceback.format_exc() @@ -70,7 +69,7 @@ def embedding_by_document(document_id, model_id, state_list=None): def exception_handler(e): ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING, State.FAILURE) - max_kb_error.error( + maxkb_error_logger.error( _('Failed to obtain vector model: {error} {traceback}').format( error=str(e), traceback=traceback.format_exc() @@ -100,11 +99,11 @@ def embedding_by_knowledge(knowledge_id, model_id): @param model_id 向量模型 :return: None """ - max_kb.info(_('Start--->Vectorized knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) + maxkb_logger.info(_('Start--->Vectorized knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) try: ListenerManagement.delete_embedding_by_knowledge(knowledge_id) document_list = QuerySet(Document).filter(knowledge_id=knowledge_id) - max_kb.info(_('Knowledge documentation: {document_names}').format( + maxkb_logger.info(_('Knowledge documentation: {document_names}').format( document_names=", ".join([d.name for d in document_list]))) for document in document_list: try: @@ -112,12 +111,12 @@ def embedding_by_knowledge(knowledge_id, model_id): except Exception as e: pass except Exception as e: - max_kb_error.error( + maxkb_error_logger.error( _('Vectorized knowledge: {knowledge_id} error {error} {traceback}'.format(knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))) finally: - max_kb.info(_('End--->Vectorized knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) + maxkb_logger.info(_('End--->Vectorized knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) def embedding_by_problem(args, model_id): diff --git a/apps/knowledge/task/generate.py b/apps/knowledge/task/generate.py index b2b9bdd23..573a06459 100644 --- a/apps/knowledge/task/generate.py +++ b/apps/knowledge/task/generate.py @@ -9,6 +9,7 @@ from langchain_core.messages import HumanMessage from common.config.embedding_config import ModelManage from common.event import ListenerManagement +from common.utils.logger import maxkb_error_logger, maxkb_logger from common.utils.page_utils import page, page_desc from knowledge.models import Paragraph, Document, Status, TaskType, State from knowledge.task.handler import save_problem @@ -16,9 +17,6 @@ from models_provider.models import Model from models_provider.tools import get_model from ops import celery_app -max_kb_error = logging.getLogger("max_kb_error") -max_kb = logging.getLogger("max_kb") - def get_llm_model(model_id): model = QuerySet(Model).filter(id=model_id).first() @@ -102,12 +100,12 @@ def generate_related_by_document_id(document_id, model_id, prompt, state_list=No ).filter(task_type_status__in=state_list, document_id=document_id) page_desc(query_set, 10, generate_problem, is_the_task_interrupted) except Exception as e: - max_kb_error.error(f'根据文档生成问题:{document_id}出现错误{str(e)}{traceback.format_exc()}') - max_kb_error.error(_('Generate issue based on document: {document_id} error {error}{traceback}').format( + maxkb_error_logger.error(f'根据文档生成问题:{document_id}出现错误{str(e)}{traceback.format_exc()}') + maxkb_error_logger.error(_('Generate issue based on document: {document_id} error {error}{traceback}').format( document_id=document_id, error=str(e), traceback=traceback.format_exc())) finally: ListenerManagement.post_update_document_status(document_id, TaskType.GENERATE_PROBLEM) - max_kb.info(_('End--->Generate problem: {document_id}').format(document_id=document_id)) + maxkb_logger.info(_('End--->Generate problem: {document_id}').format(document_id=document_id)) @celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']}, diff --git a/apps/knowledge/task/handler.py b/apps/knowledge/task/handler.py index 6069c9aa0..d4aa6dfdc 100644 --- a/apps/knowledge/task/handler.py +++ b/apps/knowledge/task/handler.py @@ -9,12 +9,10 @@ from django.db.models import QuerySet from django.utils.translation import gettext_lazy as _ from common.utils.fork import ChildLink, Fork +from common.utils.logger import maxkb_error_logger from common.utils.split_model import get_split_model from knowledge.models.knowledge import KnowledgeType, Document, Knowledge, Status -max_kb_error = logging.getLogger("max_kb_error") -max_kb = logging.getLogger("max_kb") - def get_save_handler(knowledge_id, selector): from knowledge.serializers.document import DocumentSerializers @@ -118,4 +116,4 @@ def save_problem(knowledge_id, document_id, paragraph_id, problem): } ).save(instance={"content": problem}, with_valid=True) except Exception as e: - max_kb_error.error(_('Association problem failed {error}').format(error=str(e))) + maxkb_error_logger.error(_('Association problem failed {error}').format(error=str(e))) diff --git a/apps/knowledge/task/sync.py b/apps/knowledge/task/sync.py index 63f5e26c6..22fb39511 100644 --- a/apps/knowledge/task/sync.py +++ b/apps/knowledge/task/sync.py @@ -15,10 +15,10 @@ from celery_once import QueueOnce from django.utils.translation import gettext_lazy as _ from common.utils.fork import ForkManage, Fork +from common.utils.logger import maxkb_logger, maxkb_error_logger from ops import celery_app -max_kb_error = logging.getLogger("max_kb_error") -max_kb = logging.getLogger("max_kb") + @celery_app.task(base=QueueOnce, once={'keys': ['knowledge_id']}, name='celery:sync_web_knowledge') @@ -26,15 +26,15 @@ def sync_web_knowledge(knowledge_id: str, url: str, selector: str): from knowledge.task.handler import get_save_handler try: - max_kb.info( + maxkb_logger.info( _('Start--->Start synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id)) ForkManage(url, selector.split(" ") if selector is not None else []).fork(2, set(), get_save_handler(knowledge_id, selector)) - max_kb.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id)) + maxkb_logger.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id)) except Exception as e: - max_kb_error.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format( + maxkb_error_logger.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format( knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc())) @@ -43,14 +43,14 @@ def sync_replace_web_knowledge(knowledge_id: str, url: str, selector: str): from knowledge.task.handler import get_sync_handler try: - max_kb.info( + maxkb_logger.info( _('Start--->Start synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id)) ForkManage(url, selector.split(" ") if selector is not None else []).fork(2, set(), get_sync_handler(knowledge_id )) - max_kb.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id)) + maxkb_logger.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id)) except Exception as e: - max_kb_error.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format( + maxkb_error_logger.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format( knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc())) diff --git a/apps/models_provider/impl/volcanic_engine_model_provider/model/tti.py b/apps/models_provider/impl/volcanic_engine_model_provider/model/tti.py index b07200a19..d91347ce6 100644 --- a/apps/models_provider/impl/volcanic_engine_model_provider/model/tti.py +++ b/apps/models_provider/impl/volcanic_engine_model_provider/model/tti.py @@ -18,6 +18,7 @@ from typing import Dict import requests +from common.utils.logger import maxkb_logger from models_provider.base_model_provider import MaxKBBaseModel from models_provider.impl.base_tti import BaseTextToImage @@ -35,7 +36,6 @@ req_key_dict = { 'anime_v1.3.1': 'high_aes', } -max_kb = logging.getLogger("max_kb") def sign(key, msg): @@ -60,7 +60,7 @@ def formatQuery(parameters): def signV4Request(access_key, secret_key, service, req_query, req_body): if access_key is None or secret_key is None: - max_kb.info('No access key is available.') + maxkb_logger.info('No access key is available.') sys.exit() t = datetime.datetime.utcnow() @@ -77,42 +77,36 @@ def signV4Request(access_key, secret_key, service, req_query, req_body): '\n' + 'x-date:' + current_date + '\n' canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + \ '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash - # max_kb.info(canonical_request) algorithm = 'HMAC-SHA256' credential_scope = datestamp + '/' + region + '/' + service + '/' + 'request' string_to_sign = algorithm + '\n' + current_date + '\n' + credential_scope + '\n' + hashlib.sha256( canonical_request.encode('utf-8')).hexdigest() - # max_kb.info(string_to_sign) signing_key = getSignatureKey(secret_key, datestamp, region, service) - # max_kb.info(signing_key) signature = hmac.new(signing_key, (string_to_sign).encode( 'utf-8'), hashlib.sha256).hexdigest() - # max_kb.info(signature) authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + \ credential_scope + ', ' + 'SignedHeaders=' + \ signed_headers + ', ' + 'Signature=' + signature - # max_kb.info(authorization_header) headers = {'X-Date': current_date, 'Authorization': authorization_header, 'X-Content-Sha256': payload_hash, 'Content-Type': content_type } - # max_kb.info(headers) # ************* SEND THE REQUEST ************* request_url = endpoint + '?' + canonical_querystring - max_kb.info('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++') - max_kb.info('Request URL = ' + request_url) + maxkb_logger.info('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++') + maxkb_logger.info('Request URL = ' + request_url) try: r = requests.post(request_url, headers=headers, data=req_body) except Exception as err: - max_kb.info(f'error occurred: {err}') + maxkb_logger.info(f'error occurred: {err}') raise else: - max_kb.info('\nRESPONSE++++++++++++++++++++++++++++++++++++') - max_kb.info(f'Response code: {r.status_code}\n') + maxkb_logger.info('\nRESPONSE++++++++++++++++++++++++++++++++++++') + maxkb_logger.info(f'Response code: {r.status_code}\n') # 使用 replace 方法将 \u0026 替换为 & resp_str = r.text.replace("\\u0026", "&") if r.status_code != 200: diff --git a/apps/models_provider/impl/xf_model_provider/model/stt.py b/apps/models_provider/impl/xf_model_provider/model/stt.py index 301a77bfa..50b0a54c9 100644 --- a/apps/models_provider/impl/xf_model_provider/model/stt.py +++ b/apps/models_provider/impl/xf_model_provider/model/stt.py @@ -28,7 +28,6 @@ ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE -max_kb = logging.getLogger("max_kb") class XFSparkSpeechToText(MaxKBBaseModel, BaseSpeechToText): diff --git a/apps/models_provider/impl/xf_model_provider/model/tts.py b/apps/models_provider/impl/xf_model_provider/model/tts.py index dcd51a907..bf82318c5 100644 --- a/apps/models_provider/impl/xf_model_provider/model/tts.py +++ b/apps/models_provider/impl/xf_model_provider/model/tts.py @@ -23,7 +23,6 @@ from common.utils.common import _remove_empty_lines from models_provider.base_model_provider import MaxKBBaseModel from models_provider.impl.base_tts import BaseTextToSpeech -max_kb = logging.getLogger("max_kb") STATUS_FIRST_FRAME = 0 # 第一帧的标识 STATUS_CONTINUE_FRAME = 1 # 中间帧标识 diff --git a/apps/ops/celery/logger.py b/apps/ops/celery/logger.py index bd21516b4..7e9700e01 100644 --- a/apps/ops/celery/logger.py +++ b/apps/ops/celery/logger.py @@ -8,6 +8,7 @@ from django.conf import settings from kombu import Connection, Exchange, Queue, Producer from kombu.mixins import ConsumerMixin +from common.utils.logger import maxkb_logger from .utils import get_celery_task_log_path from .const import CELERY_LOG_MAGIC_MARK @@ -209,7 +210,7 @@ class CeleryThreadTaskFileHandler(CeleryThreadingLoggerHandler): f.flush() def handle_task_start(self, task_id): - logging.getLogger("max_kb").info('handle_task_start') + maxkb_logger.info('handle_task_start') log_path = get_celery_task_log_path(task_id) thread_id = self.get_current_thread_id() self.task_id_thread_id_mapper[task_id] = thread_id @@ -217,7 +218,7 @@ class CeleryThreadTaskFileHandler(CeleryThreadingLoggerHandler): self.thread_id_fd_mapper[thread_id] = f def handle_task_end(self, task_id): - logging.getLogger('max_kb').info('handle_task_end') + maxkb_logger.info('handle_task_end') ident_id = self.task_id_thread_id_mapper.get(task_id, '') f = self.thread_id_fd_mapper.pop(ident_id, None) if f and not f.closed: diff --git a/apps/system_manage/serializers/email_setting.py b/apps/system_manage/serializers/email_setting.py index c5c50562d..98e21885e 100644 --- a/apps/system_manage/serializers/email_setting.py +++ b/apps/system_manage/serializers/email_setting.py @@ -15,6 +15,7 @@ from rest_framework import serializers from common.exception.app_exception import AppApiException from django.utils.translation import gettext_lazy as _ +from common.utils.logger import maxkb_logger from system_manage.models import SystemSetting, SettingType @@ -47,7 +48,7 @@ class EmailSettingSerializer(serializers.Serializer): self.data.get("email_use_ssl") ).open() except Exception as e: - logging.getLogger("max_kb").error(f'Exception: {e}') + maxkb_logger.error(f'Exception: {e}') raise AppApiException(1004, _('Email verification failed')) def update_or_save(self):