15308555518/ MaxKB
1
0
Fork 0
mirror of https://github.com/1Panel-dev/MaxKB.git synced 2025-12-26 01:33:05 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat: Add resource mapping table

This commit is contained in:
shaohuzhang1 2025-12-22 14:49:44 +08:00
parent a7bb173cc1
commit ba1a95af42
8 changed files with 241 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
apps/application/flow/tools.py
View File

@ -315,7 +315,8 @@ async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_
tools = await client.get_tools()
agent = create_react_agent(chat_model, tools)
recursion_limit = int(CONFIG.get("LANGCHAIN_GRAPH_RECURSION_LIMIT", '25'))
response = agent.astream({"messages": message_list}, config={"recursion_limit": recursion_limit}, stream_mode='messages')
response = agent.astream({"messages": message_list}, config={"recursion_limit": recursion_limit},
stream_mode='messages')
# 用于存储工具调用信息（按 tool_id）以及按 index 聚合分片
tool_calls_info = {}
@ -396,7 +397,6 @@ async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_
raise RuntimeError(error_msg) from None
def mcp_response_generator(chat_model, message_list, mcp_servers, mcp_output_enable=True):
"""使用全局事件循环，不创建新实例"""
result_queue = queue.Queue()
@ -427,3 +427,82 @@ def mcp_response_generator(chat_model, message_list, mcp_servers, mcp_output_ena
async def anext_async(agen):
return await agen.__anext__()
target_source_node_mapping = {
'TOOL': {'tool-lib-node': lambda n: [n.get('properties').get('node_data').get('tool_lib_id')]},
'MODEL': {'ai-chat-node': lambda n: [n.get('properties').get('node_data').get('model_id')],
'question-node': lambda n: [n.get('properties').get('node_data').get('model_id')],
'speech-to-text-node': lambda n: [n.get('properties').get('node_data').get('stt_model_id')],
'text-to-speech-node': lambda n: [n.get('properties').get('node_data').get('tts_model_id')],
'image-to-video-node': lambda n: [n.get('properties').get('node_data').get('model_id')],
'image-generate-node': lambda n: [n.get('properties').get('node_data').get('model_id')],
'intent-node': lambda n: [n.get('properties').get('node_data').get('model_id')],
'image-understand-node': lambda n: [n.get('properties').get('node_data').get('model_id')],
'parameter-extraction-node': lambda n: [n.get('properties').get('node_data').get('model_id')],
'video-understand-node': lambda n: [n.get('properties').get('node_data').get('model_id')]
},
'KNOWLEDGE': {'search-knowledge-node': lambda n: n.get('properties').get('node_data').get('knowledge_id_list')},
'APPLICATION': {
'application-node': lambda n: [n.get('properties').get('node_data').get('application_id')]
}
}
def get_node_handle_callback(source_type, source_id):
def node_handle_callback(node):
from system_manage.models.resource_mapping import ResourceMapping
response = []
for key, value in target_source_node_mapping.items():
if node.get('type') in value:
call = value.get(node.get('type'))
target_source_id_list = call(node)
for target_source_id in target_source_id_list:
if target_source_id:
response.append(ResourceMapping(source_type=source_type, target_type=key, source_id=source_id,
target_id=target_source_id))
return response
return node_handle_callback
def get_workflow_resource(workflow, node_handle):
response = []
if 'nodes' in workflow:
for node in workflow.get('nodes'):
rs = node_handle(node)
if rs:
for r in rs:
response.append(r)
if node.get('type') == 'loop-node':
r = get_workflow_resource(node.get('properties', {}).get('node_data', {}).get('loop_body'), node_handle)
for rn in r:
response.append(rn)
return list({(str(item.target_type) + str(item.target_id)): item for item in response}.values())
return []
def get_instance_resource(instance, source_type, source_id, target_type, field_call_list):
response = []
from system_manage.models.resource_mapping import ResourceMapping
for field_call in field_call_list:
target_id = field_call(instance)
if target_id:
response.append(ResourceMapping(source_type=source_type, target_type=target_type, source_id=source_id,
target_id=target_id))
return response
def save_workflow_mapping(workflow, source_type, source_id, other_resource_mapping=None):
if not other_resource_mapping:
other_resource_mapping = []
from system_manage.models.resource_mapping import ResourceMapping
from django.db.models import QuerySet
QuerySet(ResourceMapping).filter(source_type=source_type, source_id=source_id).delete()
resource_mapping_list = get_workflow_resource(workflow,
get_node_handle_callback(source_type,
source_id))
if resource_mapping_list:
resource_mapping_list += other_resource_mapping
QuerySet(ResourceMapping).bulk_create(
{(str(item.target_type) + str(item.target_id)): item for item in resource_mapping_list}.values())

2
apps/application/serializers/application.py
View File

@ -30,6 +30,7 @@ from application.flow.common import Workflow
from application.models.application import Application, ApplicationTypeChoices, ApplicationKnowledgeMapping, \
ApplicationFolder, ApplicationVersion
from application.models.application_access_token import ApplicationAccessToken
from application.serializers.common import update_resource_mapping_by_application
from common import result
from common.cache_data.application_access_token_cache import del_application_access_token
from common.database_model_manage.database_model_manage import DatabaseModelManage
@ -780,6 +781,7 @@ class ApplicationOperateSerializer(serializers.Serializer):
application_access_token.save()
else:
access_token = application_access_token.access_token
update_resource_mapping_by_application(self.data.get("application_id"))
del_application_access_token(access_token)
return self.one(with_valid=False)

23
apps/application/serializers/common.py
View File

@ -6,21 +6,18 @@
@date：2025/6/9 13:42
@desc:
"""
import json
from typing import List
from django.core.cache import cache
from django.db.models import QuerySet
from django.utils.translation import gettext_lazy as _
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from application.chat_pipeline.step.chat_step.i_chat_step import PostResponseHandler
from application.models import Application, ChatRecord, Chat, ApplicationVersion, ChatUserType, ApplicationTypeChoices, \
ApplicationKnowledgeMapping
from application.serializers.application_chat import ChatCountSerializer
from common.constants.cache_version import Cache_Version
from common.database_model_manage.database_model_manage import DatabaseModelManage
from common.encoder.encoder import SystemEncoder
from common.exception.app_exception import ChatException
from knowledge.models import Document
from models_provider.models import Model
@ -165,7 +162,7 @@ class ChatInfo:
'mcp_output_enable': self.application.mcp_output_enable,
}
def to_pipeline_manage_params(self, problem_text: str, post_response_handler: PostResponseHandler,
def to_pipeline_manage_params(self, problem_text: str, post_response_handler,
exclude_paragraph_id_list, chat_user_id: str, chat_user_type, stream=True,
form_data=None):
if form_data is None:
@ -319,3 +316,19 @@ class ChatInfo:
if chat_info_dict:
return ChatInfo.map_to_chat_info(chat_info_dict)
return None
def update_resource_mapping_by_application(application_id: str):
from application.flow.tools import get_instance_resource, save_workflow_mapping
from system_manage.models.resource_mapping import ResourceType
application = QuerySet(Application).filter(id=application_id).first()
instance_mapping = get_instance_resource(application, ResourceType.APPLICATION, str(application.id),
ResourceType.MODEL,
[lambda i: i.tts_model_id, lambda i: i.stt_model_id, ])
if application.type == 'WORK_FLOW':
save_workflow_mapping(application.work_flow, ResourceType.APPLICATION, str(application_id),
instance_mapping)
return
else:
save_workflow_mapping({}, ResourceType.APPLICATION, str(application_id),
instance_mapping)

25
apps/knowledge/serializers/common.py
View File

@ -16,6 +16,7 @@ from django.db.models import QuerySet
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from application.flow.tools import save_workflow_mapping
from common.config.embedding_config import ModelManage
from common.db.search import native_search
from common.db.sql_execute import sql_execute, update_execute
@ -23,10 +24,11 @@ from common.exception.app_exception import AppApiException
from common.utils.common import get_file_content
from common.utils.fork import Fork
from common.utils.logger import maxkb_logger
from knowledge.models import Document
from knowledge.models import Document, KnowledgeWorkflow, KnowledgeWorkflowVersion, KnowledgeType
from knowledge.models import Paragraph, Problem, ProblemParagraphMapping, Knowledge, File
from maxkb.conf import PROJECT_DIR
from models_provider.tools import get_model, get_model_default_params
from system_manage.models.resource_mapping import ResourceMapping, ResourceType
class MetaSerializer(serializers.Serializer):
@ -275,3 +277,24 @@ def drop_knowledge_index(knowledge_id=None, document_id=None):
sql = f'DROP INDEX "embedding_hnsw_idx_{k_id}"'
update_execute(sql, [])
maxkb_logger.info(f'Dropped index for knowledge ID: {k_id}')
def update_resource_mapping_by_knowledge(knowledge_id: str):
knowledge = QuerySet(Knowledge).filter(id=knowledge_id).first()
if knowledge.type == KnowledgeType.WORKFLOW:
knowledge_workflow_version = QuerySet(KnowledgeWorkflowVersion).filter(
knowledge_id=knowledge_id).order_by(
'-create_time')[0:1].first()
if knowledge_workflow_version:
other = ResourceMapping(source_type=ResourceType.KNOWLEDGE, target_type=ResourceType.MODEL,
source_id=knowledge.id,
target_id=knowledge.embedding_model_id)
save_workflow_mapping(knowledge_workflow_version.work_flow, ResourceType.KNOWLEDGE,
str(knowledge_id), [other])
return
QuerySet(ResourceMapping).filter(source_type=ResourceType.KNOWLEDGE,
source_id=knowledge.id).delete()
ResourceMapping(source_type=ResourceType.KNOWLEDGE, target_type=ResourceType.MODEL,
source_id=knowledge.id,
target_id=knowledge.embedding_model_id).save()

6
apps/knowledge/serializers/knowledge.py
View File

@ -19,6 +19,7 @@ from django.http import HttpResponse
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from application.flow.tools import get_workflow_resource, get_node_handle_callback, save_workflow_mapping
from application.models import ApplicationKnowledgeMapping
from common.config.embedding_config import VectorStore
from common.database_model_manage.database_model_manage import DatabaseModelManage
@ -34,7 +35,8 @@ from knowledge.models import Knowledge, KnowledgeScope, KnowledgeType, Document,
ProblemParagraphMapping, TaskType, State, SearchMode, KnowledgeFolder, File, Tag, KnowledgeWorkflow
from knowledge.serializers.common import ProblemParagraphManage, drop_knowledge_index, \
get_embedding_model_id_by_knowledge_id, MetaSerializer, \
GenerateRelatedSerializer, get_embedding_model_by_knowledge_id, list_paragraph, write_image, zip_dir
GenerateRelatedSerializer, get_embedding_model_by_knowledge_id, list_paragraph, write_image, zip_dir, \
update_resource_mapping_by_knowledge
from knowledge.serializers.document import DocumentSerializers
from knowledge.task.embedding import embedding_by_knowledge, delete_embedding_by_knowledge
from knowledge.task.generate import generate_related_by_knowledge_id
@ -42,6 +44,7 @@ from knowledge.task.sync import sync_web_knowledge, sync_replace_web_knowledge
from maxkb.conf import PROJECT_DIR
from models_provider.models import Model
from system_manage.models import WorkspaceUserResourcePermission, AuthTargetType
from system_manage.models.resource_mapping import ResourceType, ResourceMapping
from system_manage.serializers.user_resource_permission import UserResourcePermissionSerializer
from users.serializers.user import is_workspace_manage
@ -373,6 +376,7 @@ class KnowledgeSerializer(serializers.Serializer):
KnowledgeEditRequest(data=instance).is_valid(knowledge=knowledge)
if 'embedding_model_id' in instance:
knowledge.embedding_model_id = instance.get('embedding_model_id')
update_resource_mapping_by_knowledge(self.data.get("knowledge_id"))
if "name" in instance:
knowledge.name = instance.get("name")
if 'desc' in instance:

8
apps/knowledge/serializers/knowledge_workflow.py
View File

@ -4,6 +4,7 @@ import json
from typing import Dict
import uuid_utils.compat as uuid
from django.core.cache import cache
from django.db import transaction
from django.db.models import QuerySet
from django.utils import timezone
@ -14,6 +15,7 @@ from application.flow.common import Workflow, WorkflowMode
from application.flow.i_step_node import KnowledgeWorkflowPostHandler
from application.flow.knowledge_workflow_manage import KnowledgeWorkflowManage
from application.flow.step_node import get_node
from application.flow.tools import save_workflow_mapping
from application.serializers.application import get_mcp_tools
from common.constants.cache_version import Cache_Version
from common.db.search import page_search
@ -22,9 +24,10 @@ from common.utils.rsa_util import rsa_long_decrypt
from common.utils.tool_code import ToolExecutor
from knowledge.models import KnowledgeScope, Knowledge, KnowledgeType, KnowledgeWorkflow, KnowledgeWorkflowVersion
from knowledge.models.knowledge_action import KnowledgeAction, State
from knowledge.serializers.common import update_resource_mapping_by_knowledge
from knowledge.serializers.knowledge import KnowledgeModelSerializer
from django.core.cache import cache
from system_manage.models import AuthTargetType
from system_manage.models.resource_mapping import ResourceType
from system_manage.serializers.user_resource_permission import UserResourcePermissionSerializer
from tools.models import Tool
from users.models import User
@ -214,7 +217,7 @@ class KnowledgeWorkflowSerializer(serializers.Serializer):
)
knowledge_workflow.save()
save_workflow_mapping(instance.get('work_flow', {}), ResourceType.KNOWLEDGE, str(knowledge_id))
return {**KnowledgeModelSerializer(knowledge).data, 'document_list': []}
class Operate(serializers.Serializer):
@ -241,6 +244,7 @@ class KnowledgeWorkflowSerializer(serializers.Serializer):
QuerySet(KnowledgeWorkflow).filter(
knowledge_id=self.data.get("knowledge_id")
).update(is_publish=True, publish_time=timezone.now())
update_resource_mapping_by_knowledge(self.data.get("knowledge_id"))
return True
def edit(self, instance: Dict):

74
apps/system_manage/migrations/0005_resourcemapping.py Normal file
View File

@ -0,0 +1,74 @@
# Generated by Django 5.2.8 on 2025-12-19 09:37
import uuid_utils.compat
from django.db import migrations, models
from knowledge.models import Knowledge
def resource_mapping(apps, schema_editor):
from system_manage.models.resource_mapping import ResourceMapping
from django.db.models import QuerySet
from application.flow.tools import get_workflow_resource, get_node_handle_callback, \
get_instance_resource
from system_manage.models.resource_mapping import ResourceType
from application.models import Application
from knowledge.models import KnowledgeWorkflow
resource_mapping_list = []
for application in QuerySet(Application):
workflow_mapping = get_workflow_resource(application.work_flow,
get_node_handle_callback(ResourceType.APPLICATION,
application.id))
instance_mapping = get_instance_resource(application, ResourceType.APPLICATION, str(application.id),
ResourceType.MODEL,
[lambda i: i.tts_model_id, lambda i: i.stt_model_id, ])
resource_mapping_list += workflow_mapping
resource_mapping_list += instance_mapping
knowledge_workflow_dict = {str(kw.knowledge_id): kw for kw in QuerySet(KnowledgeWorkflow)}
for knowledge in QuerySet(Knowledge):
knowledge_workflow = knowledge_workflow_dict.get(str(knowledge.id))
if knowledge_workflow:
workflow_mapping = get_workflow_resource(knowledge_workflow.work_flow,
get_node_handle_callback(ResourceType.KNOWLEDGE,
str(knowledge_workflow.knowledge_id)))
resource_mapping_list += workflow_mapping
instance_mapping = get_instance_resource(knowledge, ResourceType.KNOWLEDGE, str(knowledge.id),
ResourceType.MODEL,
[lambda i: i.embedding_model_id])
resource_mapping_list += instance_mapping
QuerySet(ResourceMapping).bulk_create(resource_mapping_list)
class Migration(migrations.Migration):
dependencies = [
('system_manage', '0004_alter_systemsetting_type_and_more'),
('knowledge', '0007_remove_knowledgeworkflowversion_workflow_and_more'),
('application', '0003_application_stt_model_params_setting_and_more'),
]
operations = [
migrations.CreateModel(
name='ResourceMapping',
fields=[
('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')),
('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')),
('id',
models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False,
verbose_name='主键id')),
('source_type', models.CharField(
choices=[('KNOWLEDGE', '知识库'), ('APPLICATION', '应用'), ('TOOL', '工具'), ('MODEL', '模型')],
db_index=True, verbose_name='关联资源类型')),
('target_type', models.CharField(
choices=[('KNOWLEDGE', '知识库'), ('APPLICATION', '应用'), ('TOOL', '工具'), ('MODEL', '模型')],
db_index=True, verbose_name='被关联资源类型')),
('source_id', models.CharField(db_index=True, max_length=128, verbose_name='关联资源id')),
('target_id', models.CharField(db_index=True, max_length=128, verbose_name='被关联资源id')),
],
options={
'db_table': 'resource_mapping',
},
),
migrations.RunPython(resource_mapping)
]

31
apps/system_manage/models/resource_mapping.py Normal file
View File

@ -0,0 +1,31 @@
# coding=utf-8
"""
@project: MaxKB
@Author：虎虎
@file： resource_mapping.py
@date：2025/12/19 15:41
@desc:
"""
from django.db import models
import uuid_utils.compat as uuid
from common.constants.permission_constants import Group
from common.mixins.app_model_mixin import AppModelMixin
class ResourceType(models.TextChoices):
KNOWLEDGE = Group.KNOWLEDGE.value, '知识库'
APPLICATION = Group.APPLICATION.value, '应用'
TOOL = Group.TOOL.value, '工具'
MODEL = Group.MODEL.value, '模型'
class ResourceMapping(AppModelMixin):
id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id")
source_type = models.CharField(verbose_name="关联资源类型", choices=ResourceType.choices, db_index=True)
target_type = models.CharField(verbose_name="被关联资源类型", choices=ResourceType.choices, db_index=True)
source_id = models.CharField(max_length=128, verbose_name="关联资源id", db_index=True)
target_id = models.CharField(max_length=128, verbose_name="被关联资源id", db_index=True)
class Meta:
db_table = "resource_mapping"