15308555518/ MaxKB
1
0
Fork 0
mirror of https://github.com/1Panel-dev/MaxKB.git synced 2025-12-26 01:33:05 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat: add token usage and top questions statistics endpoints

This commit is contained in:
wxg0103 2025-11-13 14:56:37 +08:00
parent 74b1bce315
commit 0555632095
9 changed files with 133 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py
View File

@ -1,9 +1,7 @@
# coding=utf-8
import base64
import mimetypes
import time
from functools import reduce
from imghdr import what
from typing import List, Dict
from django.db.models import QuerySet
@ -12,7 +10,6 @@ from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, AI
from application.flow.i_step_node import NodeResult, INode
from application.flow.step_node.video_understand_step_node.i_video_understand_node import IVideoUnderstandNode
from knowledge.models import File
from models_provider.impl.volcanic_engine_model_provider.model.image import get_video_format
from models_provider.tools import get_model_instance_by_model_workspace_id

34
apps/application/serializers/application_stats.py
View File

@ -118,3 +118,37 @@ class ApplicationStatisticsSerializer(serializers.Serializer):
days.append(current_date.strftime('%Y-%m-%d'))
current_date += datetime.timedelta(days=1)
return days
def get_token_usage_statistics(self, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
start_time = self.get_start_time()
end_time = self.get_end_time()
get_token_usage = native_search(
{'default_sql': QuerySet(model=get_dynamics_model(
{'application_chat.application_id': models.UUIDField(),
'application_chat_record.create_time': models.DateTimeField()})).filter(
**{'application_chat.application_id': self.data.get('application_id'),
'application_chat_record.create_time__gte': start_time,
'application_chat_record.create_time__lte': end_time}
)},
select_string=get_file_content(
os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'get_token_usage.sql')))
return get_token_usage
def get_top_questions_statistics(self, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
start_time = self.get_start_time()
end_time = self.get_end_time()
get_top_questions = native_search(
{'default_sql': QuerySet(model=get_dynamics_model(
{'application_chat.application_id': models.UUIDField(),
'application_chat_record.create_time': models.DateTimeField()})).filter(
**{'application_chat.application_id': self.data.get('application_id'),
'application_chat_record.create_time__gte': start_time,
'application_chat_record.create_time__lte': end_time}
)},
select_string=get_file_content(
os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'top_questions.sql')))
return get_top_questions

12
apps/application/sql/get_token_usage.sql Normal file
View File

@ -0,0 +1,12 @@
SELECT
SUM(application_chat_record.message_tokens + application_chat_record.answer_tokens) as "token_usage",
COALESCE(application_chat.asker->>'username', '游客') as "username"
FROM
application_chat_record application_chat_record
LEFT JOIN application_chat application_chat ON application_chat."id" = application_chat_record.chat_id
${default_sql}
GROUP BY
COALESCE(application_chat.asker->>'username', '游客')
ORDER BY
"token_usage" DESC

11
apps/application/sql/top_questions.sql Normal file
View File

@ -0,0 +1,11 @@
SELECT COUNT(application_chat_record."id") AS chat_record_count,
COALESCE(application_chat.asker ->>'username', '游客') AS username
FROM application_chat_record application_chat_record
LEFT JOIN application_chat application_chat ON application_chat."id" = application_chat_record.chat_id
${default_sql}
GROUP BY
COALESCE (application_chat.asker->>'username', '游客')
ORDER BY
chat_record_count DESC,
username ASC

2
apps/application/urls.py
View File

@ -13,6 +13,8 @@ urlpatterns = [
path('workspace/<str:workspace_id>/application/<str:application_id>/publish', views.ApplicationAPI.Publish.as_view()),
path('workspace/<str:workspace_id>/application/<str:application_id>/application_key', views.ApplicationKey.as_view()),
path('workspace/<str:workspace_id>/application/<str:application_id>/application_stats', views.ApplicationStats.as_view()),
path('workspace/<str:workspace_id>/application/<str:application_id>/application_token_usage', views.ApplicationStats.TokenUsageStatistics.as_view()),
path('workspace/<str:workspace_id>/application/<str:application_id>/top_questions', views.ApplicationStats.TopQuestionsStatistics.as_view()),
path('workspace/<str:workspace_id>/application/<str:application_id>/application_key/<str:api_key_id>', views.ApplicationKey.Operate.as_view()),
path('workspace/<str:workspace_id>/application/<str:application_id>/export', views.ApplicationAPI.Export.as_view()),
path('workspace/<str:workspace_id>/application/<str:application_id>/application_version', views.ApplicationVersionView.as_view()),

55
apps/application/views/application_stats.py
View File

@ -46,3 +46,58 @@ class ApplicationStats(APIView):
'end_time': request.query_params.get(
'end_time')
}).get_chat_record_aggregate_trend())
class TokenUsageStatistics(APIView):
authentication_classes = [TokenAuth]
# 应用的token使用统计 根据人的使用数排序
@extend_schema(
methods=['GET'],
description=_('Application token usage statistics'),
summary=_('Application token usage statistics'),
operation_id=_('Application token usage statistics'), # type: ignore
parameters=ApplicationStatsAPI.get_parameters(),
responses=ApplicationStatsAPI.get_response(),
tags=[_('Application')] # type: ignore
)
@has_permissions(PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_application_permission(),
PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_permission_workspace_manage_role(),
ViewPermission([RoleConstants.USER.get_workspace_role()],
[PermissionConstants.APPLICATION.get_workspace_application_permission()],
CompareConstants.AND),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
def get(self, request: Request, workspace_id: str, application_id: str):
return result.success(
ApplicationStatisticsSerializer(data={'application_id': application_id, 'workspace_id': workspace_id,
'start_time': request.query_params.get(
'start_time'),
'end_time': request.query_params.get(
'end_time')
}).get_token_usage_statistics())
class TopQuestionsStatistics(APIView):
authentication_classes = [TokenAuth]
# 应用的top10问题统计
@extend_schema(
methods=['GET'],
description=_('Application top10 question statistics'),
summary=_('Application top10 question statistics'),
operation_id=_('Application top10 question statistics'), # type: ignore
parameters=ApplicationStatsAPI.get_parameters(),
responses=ApplicationStatsAPI.get_response(),
tags=[_('Application')] # type: ignore
)
@has_permissions(PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_application_permission(),
PermissionConstants.APPLICATION_OVERVIEW_READ.get_workspace_permission_workspace_manage_role(),
ViewPermission([RoleConstants.USER.get_workspace_role()],
[PermissionConstants.APPLICATION.get_workspace_application_permission()],
CompareConstants.AND),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
def get(self, request: Request, workspace_id: str, application_id: str):
return result.success(
ApplicationStatisticsSerializer(data={'application_id': application_id, 'workspace_id': workspace_id,
'start_time': request.query_params.get(
'start_time'),
'end_time': request.query_params.get(
'end_time')
}).get_top_questions_statistics())

18
apps/models_provider/impl/base_chat_open_ai.py
View File

@ -1,4 +1,5 @@
# coding=utf-8
import base64
from concurrent.futures import ThreadPoolExecutor
from requests.exceptions import ConnectTimeout, ReadTimeout
from typing import Dict, Optional, Any, Iterator, cast, Union, Sequence, Callable, Mapping
@ -211,3 +212,20 @@ class BaseChatOpenAI(ChatOpenAI):
self.usage_metadata = chat_result.response_metadata[
'token_usage'] if 'token_usage' in chat_result.response_metadata else chat_result.usage_metadata
return chat_result
def upload_file_and_get_url(self, file_stream, file_name):
"""上传文件并获取文件URL"""
base64_video = base64.b64encode(file_stream).decode("utf-8")
video_format = get_video_format(file_name)
return f'data:{video_format};base64,{base64_video}'
def get_video_format(file_name):
extension = file_name.split('.')[-1].lower()
format_map = {
'mp4': 'video/mp4',
'avi': 'video/avi',
'mov': 'video/mov',
'wmv': 'video/x-ms-wmv'
}
return format_map.get(extension, 'video/mp4')

15
apps/models_provider/impl/volcanic_engine_model_provider/model/image.py
View File

@ -25,20 +25,5 @@ class VolcanicEngineImage(MaxKBBaseModel, BaseChatOpenAI):
def is_cache_model():
return False
def upload_file_and_get_url(self, file_stream, file_name):
"""上传文件并获取文件URL"""
base64_video = base64.b64encode(file_stream).decode("utf-8")
video_format = get_video_format(file_name)
return f'data:{video_format};base64,{base64_video}'
def get_video_format(file_name):
extension = file_name.split('.')[-1].lower()
format_map = {
'mp4': 'video/mp4',
'avi': 'video/avi',
'mov': 'video/mov',
'wmv': 'video/x-ms-wmv'
}
return format_map.get(extension, 'video/mp4')

1
apps/models_provider/impl/zhipu_model_provider/model/image.py
View File

@ -3,7 +3,6 @@ from typing import Dict
from models_provider.base_model_provider import MaxKBBaseModel
from models_provider.impl.base_chat_open_ai import BaseChatOpenAI
class ZhiPuImage(MaxKBBaseModel, BaseChatOpenAI):
@staticmethod