diff --git a/apps/application/flow/tools.py b/apps/application/flow/tools.py index f8b30c2b5..9762c75a4 100644 --- a/apps/application/flow/tools.py +++ b/apps/application/flow/tools.py @@ -315,7 +315,8 @@ async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_ tools = await client.get_tools() agent = create_react_agent(chat_model, tools) recursion_limit = int(CONFIG.get("LANGCHAIN_GRAPH_RECURSION_LIMIT", '25')) - response = agent.astream({"messages": message_list}, config={"recursion_limit": recursion_limit}, stream_mode='messages') + response = agent.astream({"messages": message_list}, config={"recursion_limit": recursion_limit}, + stream_mode='messages') # 用于存储工具调用信息（按 tool_id）以及按 index 聚合分片 tool_calls_info = {} @@ -396,7 +397,6 @@ async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_ raise RuntimeError(error_msg) from None - def mcp_response_generator(chat_model, message_list, mcp_servers, mcp_output_enable=True): """使用全局事件循环，不创建新实例""" result_queue = queue.Queue() @@ -427,3 +427,82 @@ def mcp_response_generator(chat_model, message_list, mcp_servers, mcp_output_ena async def anext_async(agen): return await agen.__anext__() + + +target_source_node_mapping = { + 'TOOL': {'tool-lib-node': lambda n: [n.get('properties').get('node_data').get('tool_lib_id')]}, + 'MODEL': {'ai-chat-node': lambda n: [n.get('properties').get('node_data').get('model_id')], + 'question-node': lambda n: [n.get('properties').get('node_data').get('model_id')], + 'speech-to-text-node': lambda n: [n.get('properties').get('node_data').get('stt_model_id')], + 'text-to-speech-node': lambda n: [n.get('properties').get('node_data').get('tts_model_id')], + 'image-to-video-node': lambda n: [n.get('properties').get('node_data').get('model_id')], + 'image-generate-node': lambda n: [n.get('properties').get('node_data').get('model_id')], + 'intent-node': lambda n: [n.get('properties').get('node_data').get('model_id')], + 'image-understand-node': lambda n: [n.get('properties').get('node_data').get('model_id')], + 'parameter-extraction-node': lambda n: [n.get('properties').get('node_data').get('model_id')], + 'video-understand-node': lambda n: [n.get('properties').get('node_data').get('model_id')] + }, + 'KNOWLEDGE': {'search-knowledge-node': lambda n: n.get('properties').get('node_data').get('knowledge_id_list')}, + 'APPLICATION': { + 'application-node': lambda n: [n.get('properties').get('node_data').get('application_id')] + } +} + + +def get_node_handle_callback(source_type, source_id): + def node_handle_callback(node): + from system_manage.models.resource_mapping import ResourceMapping + response = [] + for key, value in target_source_node_mapping.items(): + if node.get('type') in value: + call = value.get(node.get('type')) + target_source_id_list = call(node) + for target_source_id in target_source_id_list: + if target_source_id: + response.append(ResourceMapping(source_type=source_type, target_type=key, source_id=source_id, + target_id=target_source_id)) + return response + + return node_handle_callback + + +def get_workflow_resource(workflow, node_handle): + response = [] + if 'nodes' in workflow: + for node in workflow.get('nodes'): + rs = node_handle(node) + if rs: + for r in rs: + response.append(r) + if node.get('type') == 'loop-node': + r = get_workflow_resource(node.get('properties', {}).get('node_data', {}).get('loop_body'), node_handle) + for rn in r: + response.append(rn) + return list({(str(item.target_type) + str(item.target_id)): item for item in response}.values()) + return [] + + +def get_instance_resource(instance, source_type, source_id, target_type, field_call_list): + response = [] + from system_manage.models.resource_mapping import ResourceMapping + for field_call in field_call_list: + target_id = field_call(instance) + if target_id: + response.append(ResourceMapping(source_type=source_type, target_type=target_type, source_id=source_id, + target_id=target_id)) + return response + + +def save_workflow_mapping(workflow, source_type, source_id, other_resource_mapping=None): + if not other_resource_mapping: + other_resource_mapping = [] + from system_manage.models.resource_mapping import ResourceMapping + from django.db.models import QuerySet + QuerySet(ResourceMapping).filter(source_type=source_type, source_id=source_id).delete() + resource_mapping_list = get_workflow_resource(workflow, + get_node_handle_callback(source_type, + source_id)) + if resource_mapping_list: + resource_mapping_list += other_resource_mapping + QuerySet(ResourceMapping).bulk_create( + {(str(item.target_type) + str(item.target_id)): item for item in resource_mapping_list}.values()) diff --git a/apps/application/serializers/application.py b/apps/application/serializers/application.py index dfa5f897c..00eea9fac 100644 --- a/apps/application/serializers/application.py +++ b/apps/application/serializers/application.py @@ -30,6 +30,7 @@ from application.flow.common import Workflow from application.models.application import Application, ApplicationTypeChoices, ApplicationKnowledgeMapping, \ ApplicationFolder, ApplicationVersion from application.models.application_access_token import ApplicationAccessToken +from application.serializers.common import update_resource_mapping_by_application from common import result from common.cache_data.application_access_token_cache import del_application_access_token from common.database_model_manage.database_model_manage import DatabaseModelManage @@ -780,6 +781,7 @@ class ApplicationOperateSerializer(serializers.Serializer): application_access_token.save() else: access_token = application_access_token.access_token + update_resource_mapping_by_application(self.data.get("application_id")) del_application_access_token(access_token) return self.one(with_valid=False) diff --git a/apps/application/serializers/common.py b/apps/application/serializers/common.py index b7ac4c5e4..d476ed7c0 100644 --- a/apps/application/serializers/common.py +++ b/apps/application/serializers/common.py @@ -6,21 +6,18 @@ @date：2025/6/9 13:42 @desc: """ -import json from typing import List from django.core.cache import cache from django.db.models import QuerySet -from django.utils.translation import gettext_lazy as _ from django.utils import timezone +from django.utils.translation import gettext_lazy as _ -from application.chat_pipeline.step.chat_step.i_chat_step import PostResponseHandler from application.models import Application, ChatRecord, Chat, ApplicationVersion, ChatUserType, ApplicationTypeChoices, \ ApplicationKnowledgeMapping from application.serializers.application_chat import ChatCountSerializer from common.constants.cache_version import Cache_Version from common.database_model_manage.database_model_manage import DatabaseModelManage -from common.encoder.encoder import SystemEncoder from common.exception.app_exception import ChatException from knowledge.models import Document from models_provider.models import Model @@ -165,7 +162,7 @@ class ChatInfo: 'mcp_output_enable': self.application.mcp_output_enable, } - def to_pipeline_manage_params(self, problem_text: str, post_response_handler: PostResponseHandler, + def to_pipeline_manage_params(self, problem_text: str, post_response_handler, exclude_paragraph_id_list, chat_user_id: str, chat_user_type, stream=True, form_data=None): if form_data is None: @@ -319,3 +316,19 @@ class ChatInfo: if chat_info_dict: return ChatInfo.map_to_chat_info(chat_info_dict) return None + + +def update_resource_mapping_by_application(application_id: str): + from application.flow.tools import get_instance_resource, save_workflow_mapping + from system_manage.models.resource_mapping import ResourceType + application = QuerySet(Application).filter(id=application_id).first() + instance_mapping = get_instance_resource(application, ResourceType.APPLICATION, str(application.id), + ResourceType.MODEL, + [lambda i: i.tts_model_id, lambda i: i.stt_model_id, ]) + if application.type == 'WORK_FLOW': + save_workflow_mapping(application.work_flow, ResourceType.APPLICATION, str(application_id), + instance_mapping) + return + else: + save_workflow_mapping({}, ResourceType.APPLICATION, str(application_id), + instance_mapping) diff --git a/apps/knowledge/serializers/common.py b/apps/knowledge/serializers/common.py index a76769cba..1980d1865 100644 --- a/apps/knowledge/serializers/common.py +++ b/apps/knowledge/serializers/common.py @@ -16,6 +16,7 @@ from django.db.models import QuerySet from django.utils.translation import gettext_lazy as _ from rest_framework import serializers +from application.flow.tools import save_workflow_mapping from common.config.embedding_config import ModelManage from common.db.search import native_search from common.db.sql_execute import sql_execute, update_execute @@ -23,10 +24,11 @@ from common.exception.app_exception import AppApiException from common.utils.common import get_file_content from common.utils.fork import Fork from common.utils.logger import maxkb_logger -from knowledge.models import Document +from knowledge.models import Document, KnowledgeWorkflow, KnowledgeWorkflowVersion, KnowledgeType from knowledge.models import Paragraph, Problem, ProblemParagraphMapping, Knowledge, File from maxkb.conf import PROJECT_DIR from models_provider.tools import get_model, get_model_default_params +from system_manage.models.resource_mapping import ResourceMapping, ResourceType class MetaSerializer(serializers.Serializer): @@ -275,3 +277,24 @@ def drop_knowledge_index(knowledge_id=None, document_id=None): sql = f'DROP INDEX "embedding_hnsw_idx_{k_id}"' update_execute(sql, []) maxkb_logger.info(f'Dropped index for knowledge ID: {k_id}') + + +def update_resource_mapping_by_knowledge(knowledge_id: str): + knowledge = QuerySet(Knowledge).filter(id=knowledge_id).first() + if knowledge.type == KnowledgeType.WORKFLOW: + knowledge_workflow_version = QuerySet(KnowledgeWorkflowVersion).filter( + knowledge_id=knowledge_id).order_by( + '-create_time')[0:1].first() + if knowledge_workflow_version: + other = ResourceMapping(source_type=ResourceType.KNOWLEDGE, target_type=ResourceType.MODEL, + source_id=knowledge.id, + target_id=knowledge.embedding_model_id) + save_workflow_mapping(knowledge_workflow_version.work_flow, ResourceType.KNOWLEDGE, + str(knowledge_id), [other]) + return + + QuerySet(ResourceMapping).filter(source_type=ResourceType.KNOWLEDGE, + source_id=knowledge.id).delete() + ResourceMapping(source_type=ResourceType.KNOWLEDGE, target_type=ResourceType.MODEL, + source_id=knowledge.id, + target_id=knowledge.embedding_model_id).save() diff --git a/apps/knowledge/serializers/knowledge.py b/apps/knowledge/serializers/knowledge.py index 61f1e38ff..cbaffd3fb 100644 --- a/apps/knowledge/serializers/knowledge.py +++ b/apps/knowledge/serializers/knowledge.py @@ -19,6 +19,7 @@ from django.http import HttpResponse from django.utils.translation import gettext_lazy as _ from rest_framework import serializers +from application.flow.tools import get_workflow_resource, get_node_handle_callback, save_workflow_mapping from application.models import ApplicationKnowledgeMapping from common.config.embedding_config import VectorStore from common.database_model_manage.database_model_manage import DatabaseModelManage @@ -34,7 +35,8 @@ from knowledge.models import Knowledge, KnowledgeScope, KnowledgeType, Document, ProblemParagraphMapping, TaskType, State, SearchMode, KnowledgeFolder, File, Tag, KnowledgeWorkflow from knowledge.serializers.common import ProblemParagraphManage, drop_knowledge_index, \ get_embedding_model_id_by_knowledge_id, MetaSerializer, \ - GenerateRelatedSerializer, get_embedding_model_by_knowledge_id, list_paragraph, write_image, zip_dir + GenerateRelatedSerializer, get_embedding_model_by_knowledge_id, list_paragraph, write_image, zip_dir, \ + update_resource_mapping_by_knowledge from knowledge.serializers.document import DocumentSerializers from knowledge.task.embedding import embedding_by_knowledge, delete_embedding_by_knowledge from knowledge.task.generate import generate_related_by_knowledge_id @@ -42,6 +44,7 @@ from knowledge.task.sync import sync_web_knowledge, sync_replace_web_knowledge from maxkb.conf import PROJECT_DIR from models_provider.models import Model from system_manage.models import WorkspaceUserResourcePermission, AuthTargetType +from system_manage.models.resource_mapping import ResourceType, ResourceMapping from system_manage.serializers.user_resource_permission import UserResourcePermissionSerializer from users.serializers.user import is_workspace_manage @@ -373,6 +376,7 @@ class KnowledgeSerializer(serializers.Serializer): KnowledgeEditRequest(data=instance).is_valid(knowledge=knowledge) if 'embedding_model_id' in instance: knowledge.embedding_model_id = instance.get('embedding_model_id') + update_resource_mapping_by_knowledge(self.data.get("knowledge_id")) if "name" in instance: knowledge.name = instance.get("name") if 'desc' in instance: diff --git a/apps/knowledge/serializers/knowledge_workflow.py b/apps/knowledge/serializers/knowledge_workflow.py index 245869b08..930dcbe1a 100644 --- a/apps/knowledge/serializers/knowledge_workflow.py +++ b/apps/knowledge/serializers/knowledge_workflow.py @@ -4,6 +4,7 @@ import json from typing import Dict import uuid_utils.compat as uuid +from django.core.cache import cache from django.db import transaction from django.db.models import QuerySet from django.utils import timezone @@ -14,6 +15,7 @@ from application.flow.common import Workflow, WorkflowMode from application.flow.i_step_node import KnowledgeWorkflowPostHandler from application.flow.knowledge_workflow_manage import KnowledgeWorkflowManage from application.flow.step_node import get_node +from application.flow.tools import save_workflow_mapping from application.serializers.application import get_mcp_tools from common.constants.cache_version import Cache_Version from common.db.search import page_search @@ -22,9 +24,10 @@ from common.utils.rsa_util import rsa_long_decrypt from common.utils.tool_code import ToolExecutor from knowledge.models import KnowledgeScope, Knowledge, KnowledgeType, KnowledgeWorkflow, KnowledgeWorkflowVersion from knowledge.models.knowledge_action import KnowledgeAction, State +from knowledge.serializers.common import update_resource_mapping_by_knowledge from knowledge.serializers.knowledge import KnowledgeModelSerializer -from django.core.cache import cache from system_manage.models import AuthTargetType +from system_manage.models.resource_mapping import ResourceType from system_manage.serializers.user_resource_permission import UserResourcePermissionSerializer from tools.models import Tool from users.models import User @@ -214,7 +217,7 @@ class KnowledgeWorkflowSerializer(serializers.Serializer): ) knowledge_workflow.save() - + save_workflow_mapping(instance.get('work_flow', {}), ResourceType.KNOWLEDGE, str(knowledge_id)) return {**KnowledgeModelSerializer(knowledge).data, 'document_list': []} class Operate(serializers.Serializer): @@ -241,6 +244,7 @@ class KnowledgeWorkflowSerializer(serializers.Serializer): QuerySet(KnowledgeWorkflow).filter( knowledge_id=self.data.get("knowledge_id") ).update(is_publish=True, publish_time=timezone.now()) + update_resource_mapping_by_knowledge(self.data.get("knowledge_id")) return True def edit(self, instance: Dict): diff --git a/apps/system_manage/migrations/0005_resourcemapping.py b/apps/system_manage/migrations/0005_resourcemapping.py new file mode 100644 index 000000000..8e8e76b72 --- /dev/null +++ b/apps/system_manage/migrations/0005_resourcemapping.py @@ -0,0 +1,74 @@ +# Generated by Django 5.2.8 on 2025-12-19 09:37 + +import uuid_utils.compat +from django.db import migrations, models + +from knowledge.models import Knowledge + + +def resource_mapping(apps, schema_editor): + from system_manage.models.resource_mapping import ResourceMapping + from django.db.models import QuerySet + from application.flow.tools import get_workflow_resource, get_node_handle_callback, \ + get_instance_resource + from system_manage.models.resource_mapping import ResourceType + from application.models import Application + from knowledge.models import KnowledgeWorkflow + resource_mapping_list = [] + for application in QuerySet(Application): + workflow_mapping = get_workflow_resource(application.work_flow, + get_node_handle_callback(ResourceType.APPLICATION, + application.id)) + instance_mapping = get_instance_resource(application, ResourceType.APPLICATION, str(application.id), + ResourceType.MODEL, + [lambda i: i.tts_model_id, lambda i: i.stt_model_id, ]) + resource_mapping_list += workflow_mapping + resource_mapping_list += instance_mapping + knowledge_workflow_dict = {str(kw.knowledge_id): kw for kw in QuerySet(KnowledgeWorkflow)} + for knowledge in QuerySet(Knowledge): + knowledge_workflow = knowledge_workflow_dict.get(str(knowledge.id)) + if knowledge_workflow: + workflow_mapping = get_workflow_resource(knowledge_workflow.work_flow, + get_node_handle_callback(ResourceType.KNOWLEDGE, + str(knowledge_workflow.knowledge_id))) + resource_mapping_list += workflow_mapping + instance_mapping = get_instance_resource(knowledge, ResourceType.KNOWLEDGE, str(knowledge.id), + ResourceType.MODEL, + [lambda i: i.embedding_model_id]) + + resource_mapping_list += instance_mapping + + QuerySet(ResourceMapping).bulk_create(resource_mapping_list) + + +class Migration(migrations.Migration): + dependencies = [ + ('system_manage', '0004_alter_systemsetting_type_and_more'), + ('knowledge', '0007_remove_knowledgeworkflowversion_workflow_and_more'), + ('application', '0003_application_stt_model_params_setting_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='ResourceMapping', + fields=[ + ('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')), + ('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')), + ('id', + models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, + verbose_name='主键id')), + ('source_type', models.CharField( + choices=[('KNOWLEDGE', '知识库'), ('APPLICATION', '应用'), ('TOOL', '工具'), ('MODEL', '模型')], + db_index=True, verbose_name='关联资源类型')), + ('target_type', models.CharField( + choices=[('KNOWLEDGE', '知识库'), ('APPLICATION', '应用'), ('TOOL', '工具'), ('MODEL', '模型')], + db_index=True, verbose_name='被关联资源类型')), + ('source_id', models.CharField(db_index=True, max_length=128, verbose_name='关联资源id')), + ('target_id', models.CharField(db_index=True, max_length=128, verbose_name='被关联资源id')), + ], + options={ + 'db_table': 'resource_mapping', + }, + ), + migrations.RunPython(resource_mapping) + ] diff --git a/apps/system_manage/models/resource_mapping.py b/apps/system_manage/models/resource_mapping.py new file mode 100644 index 000000000..5d38092a8 --- /dev/null +++ b/apps/system_manage/models/resource_mapping.py @@ -0,0 +1,31 @@ +# coding=utf-8 +""" + @project: MaxKB + @Author：虎虎 + @file： resource_mapping.py + @date：2025/12/19 15:41 + @desc: +""" +from django.db import models +import uuid_utils.compat as uuid + +from common.constants.permission_constants import Group +from common.mixins.app_model_mixin import AppModelMixin + + +class ResourceType(models.TextChoices): + KNOWLEDGE = Group.KNOWLEDGE.value, '知识库' + APPLICATION = Group.APPLICATION.value, '应用' + TOOL = Group.TOOL.value, '工具' + MODEL = Group.MODEL.value, '模型' + + +class ResourceMapping(AppModelMixin): + id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id") + source_type = models.CharField(verbose_name="关联资源类型", choices=ResourceType.choices, db_index=True) + target_type = models.CharField(verbose_name="被关联资源类型", choices=ResourceType.choices, db_index=True) + source_id = models.CharField(max_length=128, verbose_name="关联资源id", db_index=True) + target_id = models.CharField(max_length=128, verbose_name="被关联资源id", db_index=True) + + class Meta: + db_table = "resource_mapping"