From 0555632095b696ad3490e31c8997c76ce43ffcc9 Mon Sep 17 00:00:00 2001 From: wxg0103 <727495428@qq.com> Date: Thu, 13 Nov 2025 14:56:37 +0800 Subject: [PATCH] feat: add token usage and top questions statistics endpoints --- .../impl/base_video_understand_node.py | 5 +- .../serializers/application_stats.py | 34 ++++++++++++ apps/application/sql/get_token_usage.sql | 12 ++++ apps/application/sql/top_questions.sql | 11 ++++ apps/application/urls.py | 2 + apps/application/views/application_stats.py | 55 +++++++++++++++++++ .../models_provider/impl/base_chat_open_ai.py | 18 ++++++ .../model/image.py | 15 ----- .../impl/zhipu_model_provider/model/image.py | 1 - 9 files changed, 133 insertions(+), 20 deletions(-) create mode 100644 apps/application/sql/get_token_usage.sql create mode 100644 apps/application/sql/top_questions.sql diff --git a/apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py b/apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py index 2118e6a9f..67ca26170 100644 --- a/apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py +++ b/apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py @@ -1,9 +1,7 @@ # coding=utf-8 -import base64 -import mimetypes + import time from functools import reduce -from imghdr import what from typing import List, Dict from django.db.models import QuerySet @@ -12,7 +10,6 @@ from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, AI from application.flow.i_step_node import NodeResult, INode from application.flow.step_node.video_understand_step_node.i_video_understand_node import IVideoUnderstandNode from knowledge.models import File -from models_provider.impl.volcanic_engine_model_provider.model.image import get_video_format from models_provider.tools import get_model_instance_by_model_workspace_id diff --git a/apps/application/serializers/application_stats.py b/apps/application/serializers/application_stats.py index 3be5f211f..8d45a5311 100644 --- a/apps/application/serializers/application_stats.py +++ b/apps/application/serializers/application_stats.py @@ -118,3 +118,37 @@ class ApplicationStatisticsSerializer(serializers.Serializer): days.append(current_date.strftime('%Y-%m-%d')) current_date += datetime.timedelta(days=1) return days + + def get_token_usage_statistics(self, with_valid=True): + if with_valid: + self.is_valid(raise_exception=True) + start_time = self.get_start_time() + end_time = self.get_end_time() + get_token_usage = native_search( + {'default_sql': QuerySet(model=get_dynamics_model( + {'application_chat.application_id': models.UUIDField(), + 'application_chat_record.create_time': models.DateTimeField()})).filter( + **{'application_chat.application_id': self.data.get('application_id'), + 'application_chat_record.create_time__gte': start_time, + 'application_chat_record.create_time__lte': end_time} + )}, + select_string=get_file_content( + os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'get_token_usage.sql'))) + return get_token_usage + + def get_top_questions_statistics(self, with_valid=True): + if with_valid: + self.is_valid(raise_exception=True) + start_time = self.get_start_time() + end_time = self.get_end_time() + get_top_questions = native_search( + {'default_sql': QuerySet(model=get_dynamics_model( + {'application_chat.application_id': models.UUIDField(), + 'application_chat_record.create_time': models.DateTimeField()})).filter( + **{'application_chat.application_id': self.data.get('application_id'), + 'application_chat_record.create_time__gte': start_time, + 'application_chat_record.create_time__lte': end_time} + )}, + select_string=get_file_content( + os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'top_questions.sql'))) + return get_top_questions \ No newline at end of file diff --git a/apps/application/sql/get_token_usage.sql b/apps/application/sql/get_token_usage.sql new file mode 100644 index 000000000..848a36430 --- /dev/null +++ b/apps/application/sql/get_token_usage.sql @@ -0,0 +1,12 @@ + +SELECT + SUM(application_chat_record.message_tokens + application_chat_record.answer_tokens) as "token_usage", + COALESCE(application_chat.asker->>'username', '游客') as "username" +FROM + application_chat_record application_chat_record + LEFT JOIN application_chat application_chat ON application_chat."id" = application_chat_record.chat_id +${default_sql} +GROUP BY + COALESCE(application_chat.asker->>'username', '游客') +ORDER BY + "token_usage" DESC diff --git a/apps/application/sql/top_questions.sql b/apps/application/sql/top_questions.sql new file mode 100644 index 000000000..5997934ec --- /dev/null +++ b/apps/application/sql/top_questions.sql @@ -0,0 +1,11 @@ +SELECT COUNT(application_chat_record."id") AS chat_record_count, + COALESCE(application_chat.asker ->>'username', '游客') AS username +FROM application_chat_record application_chat_record + LEFT JOIN application_chat application_chat ON application_chat."id" = application_chat_record.chat_id + ${default_sql} +GROUP BY + COALESCE (application_chat.asker->>'username', '游客') +ORDER BY + chat_record_count DESC, + username ASC + diff --git a/apps/application/urls.py b/apps/application/urls.py index fb6a8ac3d..34ded9fe0 100644 --- a/apps/application/urls.py +++ b/apps/application/urls.py @@ -13,6 +13,8 @@ urlpatterns = [ path('workspace//application//publish', views.ApplicationAPI.Publish.as_view()), path('workspace//application//application_key', views.ApplicationKey.as_view()), path('workspace//application//application_stats', views.ApplicationStats.as_view()), + path('workspace//application//application_token_usage', views.ApplicationStats.TokenUsageStatistics.as_view()), + path('workspace//application//top_questions', views.ApplicationStats.TopQuestionsStatistics.as_view()), path('workspace//application//application_key/', views.ApplicationKey.Operate.as_view()), path('workspace//application//export', views.ApplicationAPI.Export.as_view()), path('workspace//application//application_version', views.ApplicationVersionView.as_view()), diff --git a/apps/application/views/application_stats.py b/apps/application/views/application_stats.py index 17b43fe37..a56ba8b12 100644 --- a/apps/application/views/application_stats.py +++ b/apps/application/views/application_stats.py @@ -46,3 +46,58 @@ class ApplicationStats(APIView): 'end_time': request.query_params.get( 'end_time') }).get_chat_record_aggregate_trend()) + + class TokenUsageStatistics(APIView): + authentication_classes = [TokenAuth] + + # 应用的token使用统计 根据人的使用数排序 + @extend_schema( + methods=['GET'], + description=_('Application token usage statistics'), + summary=_('Application token usage statistics'), + operation_id=_('Application token usage statistics'), # type: ignore + parameters=ApplicationStatsAPI.get_parameters(), + responses=ApplicationStatsAPI.get_response(), + tags=[_('Application')] # type: ignore + ) + @has_permissions(PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_application_permission(), + PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_permission_workspace_manage_role(), + ViewPermission([RoleConstants.USER.get_workspace_role()], + [PermissionConstants.APPLICATION.get_workspace_application_permission()], + CompareConstants.AND), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role()) + def get(self, request: Request, workspace_id: str, application_id: str): + return result.success( + ApplicationStatisticsSerializer(data={'application_id': application_id, 'workspace_id': workspace_id, + 'start_time': request.query_params.get( + 'start_time'), + 'end_time': request.query_params.get( + 'end_time') + }).get_token_usage_statistics()) + + class TopQuestionsStatistics(APIView): + authentication_classes = [TokenAuth] + # 应用的top10问题统计 + @extend_schema( + methods=['GET'], + description=_('Application top10 question statistics'), + summary=_('Application top10 question statistics'), + operation_id=_('Application top10 question statistics'), # type: ignore + parameters=ApplicationStatsAPI.get_parameters(), + responses=ApplicationStatsAPI.get_response(), + tags=[_('Application')] # type: ignore + ) + @has_permissions(PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_application_permission(), + PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_permission_workspace_manage_role(), + ViewPermission([RoleConstants.USER.get_workspace_role()], + [PermissionConstants.APPLICATION.get_workspace_application_permission()], + CompareConstants.AND), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role()) + def get(self, request: Request, workspace_id: str, application_id: str): + return result.success( + ApplicationStatisticsSerializer(data={'application_id': application_id, 'workspace_id': workspace_id, + 'start_time': request.query_params.get( + 'start_time'), + 'end_time': request.query_params.get( + 'end_time') + }).get_top_questions_statistics()) diff --git a/apps/models_provider/impl/base_chat_open_ai.py b/apps/models_provider/impl/base_chat_open_ai.py index 6c6698d12..b4959f298 100644 --- a/apps/models_provider/impl/base_chat_open_ai.py +++ b/apps/models_provider/impl/base_chat_open_ai.py @@ -1,4 +1,5 @@ # coding=utf-8 +import base64 from concurrent.futures import ThreadPoolExecutor from requests.exceptions import ConnectTimeout, ReadTimeout from typing import Dict, Optional, Any, Iterator, cast, Union, Sequence, Callable, Mapping @@ -211,3 +212,20 @@ class BaseChatOpenAI(ChatOpenAI): self.usage_metadata = chat_result.response_metadata[ 'token_usage'] if 'token_usage' in chat_result.response_metadata else chat_result.usage_metadata return chat_result + + + def upload_file_and_get_url(self, file_stream, file_name): + """上传文件并获取文件URL""" + base64_video = base64.b64encode(file_stream).decode("utf-8") + video_format = get_video_format(file_name) + return f'data:{video_format};base64,{base64_video}' + +def get_video_format(file_name): + extension = file_name.split('.')[-1].lower() + format_map = { + 'mp4': 'video/mp4', + 'avi': 'video/avi', + 'mov': 'video/mov', + 'wmv': 'video/x-ms-wmv' + } + return format_map.get(extension, 'video/mp4') \ No newline at end of file diff --git a/apps/models_provider/impl/volcanic_engine_model_provider/model/image.py b/apps/models_provider/impl/volcanic_engine_model_provider/model/image.py index 05ce9d933..73699d826 100644 --- a/apps/models_provider/impl/volcanic_engine_model_provider/model/image.py +++ b/apps/models_provider/impl/volcanic_engine_model_provider/model/image.py @@ -25,20 +25,5 @@ class VolcanicEngineImage(MaxKBBaseModel, BaseChatOpenAI): def is_cache_model(): return False - def upload_file_and_get_url(self, file_stream, file_name): - """上传文件并获取文件URL""" - base64_video = base64.b64encode(file_stream).decode("utf-8") - video_format = get_video_format(file_name) - return f'data:{video_format};base64,{base64_video}' - -def get_video_format(file_name): - extension = file_name.split('.')[-1].lower() - format_map = { - 'mp4': 'video/mp4', - 'avi': 'video/avi', - 'mov': 'video/mov', - 'wmv': 'video/x-ms-wmv' - } - return format_map.get(extension, 'video/mp4') diff --git a/apps/models_provider/impl/zhipu_model_provider/model/image.py b/apps/models_provider/impl/zhipu_model_provider/model/image.py index d470af1ba..cb4bae6b4 100644 --- a/apps/models_provider/impl/zhipu_model_provider/model/image.py +++ b/apps/models_provider/impl/zhipu_model_provider/model/image.py @@ -3,7 +3,6 @@ from typing import Dict from models_provider.base_model_provider import MaxKBBaseModel from models_provider.impl.base_chat_open_ai import BaseChatOpenAI - class ZhiPuImage(MaxKBBaseModel, BaseChatOpenAI): @staticmethod