diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..2a5e6736d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + timezone: "Asia/Shanghai" + day: "friday" + target-branch: "v2" \ No newline at end of file diff --git a/README.md b/README.md index f365fb37b..06025c069 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ MaxKB = Max Knowledge Brain, it is a powerful and easy-to-use AI assistant that integrates Retrieval-Augmented Generation (RAG) pipelines, supports robust workflows, and provides advanced MCP tool-use capabilities. MaxKB is widely applied in scenarios such as intelligent customer service, corporate internal knowledge bases, academic research, and education. - **RAG Pipeline**: Supports direct uploading of documents / automatic crawling of online documents, with features for automatic text splitting, vectorization. This effectively reduces hallucinations in large models, providing a superior smart Q&A interaction experience. -- **Flexible Orchestration**: Equipped with a powerful workflow engine, function library and MCP tool-use, enabling the orchestration of AI processes to meet the needs of complex business scenarios. +- **Agentic Workflow**: Equipped with a powerful workflow engine, function library and MCP tool-use, enabling the orchestration of AI processes to meet the needs of complex business scenarios. - **Seamless Integration**: Facilitates zero-coding rapid integration into third-party business systems, quickly equipping existing systems with intelligent Q&A capabilities to enhance user satisfaction. - **Model-Agnostic**: Supports various large models, including private models (such as DeepSeek, Llama, Qwen, etc.) and public models (like OpenAI, Claude, Gemini, etc.). - **Multi Modal**: Native support for input and output text, image, audio and video. diff --git a/apps/application/flow/step_node/start_node/impl/base_start_node.py b/apps/application/flow/step_node/start_node/impl/base_start_node.py index bf5203274..bd5bcbeee 100644 --- a/apps/application/flow/step_node/start_node/impl/base_start_node.py +++ b/apps/application/flow/step_node/start_node/impl/base_start_node.py @@ -40,6 +40,7 @@ class BaseStartStepNode(IStarNode): self.context['document'] = details.get('document_list') self.context['image'] = details.get('image_list') self.context['audio'] = details.get('audio_list') + self.context['other'] = details.get('other_list') self.status = details.get('status') self.err_message = details.get('err_message') for key, value in workflow_variable.items(): @@ -59,7 +60,8 @@ class BaseStartStepNode(IStarNode): 'question': question, 'image': self.workflow_manage.image_list, 'document': self.workflow_manage.document_list, - 'audio': self.workflow_manage.audio_list + 'audio': self.workflow_manage.audio_list, + 'other': self.workflow_manage.other_list, } return NodeResult(node_variable, workflow_variable) @@ -83,5 +85,6 @@ class BaseStartStepNode(IStarNode): 'image_list': self.context.get('image'), 'document_list': self.context.get('document'), 'audio_list': self.context.get('audio'), + 'other_list': self.context.get('other'), 'global_fields': global_fields } diff --git a/apps/application/flow/workflow_manage.py b/apps/application/flow/workflow_manage.py index be91f69be..0f7bc9c75 100644 --- a/apps/application/flow/workflow_manage.py +++ b/apps/application/flow/workflow_manage.py @@ -238,6 +238,7 @@ class WorkflowManage: base_to_response: BaseToResponse = SystemToResponse(), form_data=None, image_list=None, document_list=None, audio_list=None, + other_list=None, start_node_id=None, start_node_data=None, chat_record=None, child_node=None): if form_data is None: @@ -248,12 +249,15 @@ class WorkflowManage: document_list = [] if audio_list is None: audio_list = [] + if other_list is None: + other_list = [] self.start_node_id = start_node_id self.start_node = None self.form_data = form_data self.image_list = image_list self.document_list = document_list self.audio_list = audio_list + self.other_list = other_list self.params = params self.flow = flow self.context = {} diff --git a/apps/application/models/application.py b/apps/application/models/application.py index dfe9534e8..0032271a7 100644 --- a/apps/application/models/application.py +++ b/apps/application/models/application.py @@ -11,7 +11,7 @@ import uuid from django.contrib.postgres.fields import ArrayField from django.db import models from langchain.schema import HumanMessage, AIMessage - +from django.utils.translation import gettext as _ from common.encoder.encoder import SystemEncoder from common.mixins.app_model_mixin import AppModelMixin from dataset.models.data_set import DataSet @@ -167,7 +167,11 @@ class ChatRecord(AppModelMixin): return HumanMessage(content=self.problem_text) def get_ai_message(self): - return AIMessage(content=self.answer_text) + answer_text = self.answer_text + if answer_text is None or len(str(answer_text).strip()) == 0: + answer_text = _( + 'Sorry, no relevant content was found. Please re-describe your problem or provide more information. ') + return AIMessage(content=answer_text) def get_node_details_runtime_node_id(self, runtime_node_id): return self.details.get(runtime_node_id, None) diff --git a/apps/application/serializers/chat_message_serializers.py b/apps/application/serializers/chat_message_serializers.py index 2194028e6..2ce5273f7 100644 --- a/apps/application/serializers/chat_message_serializers.py +++ b/apps/application/serializers/chat_message_serializers.py @@ -213,12 +213,21 @@ class OpenAIChatSerializer(serializers.Serializer): return instance.get('messages')[-1].get('content') @staticmethod - def generate_chat(chat_id, application_id, message, client_id): + def generate_chat(chat_id, application_id, message, client_id, asker=None): if chat_id is None: chat_id = str(uuid.uuid1()) chat = QuerySet(Chat).filter(id=chat_id).first() if chat is None: - Chat(id=chat_id, application_id=application_id, abstract=message[0:1024], client_id=client_id).save() + asker_dict = {'user_name': '游客'} + if asker is not None: + if isinstance(asker, str): + asker_dict = { + 'user_name': asker + } + elif isinstance(asker, dict): + asker_dict = asker + Chat(id=chat_id, application_id=application_id, abstract=message[0:1024], client_id=client_id, + asker=asker_dict).save() return chat_id def chat(self, instance: Dict, with_valid=True): @@ -232,7 +241,8 @@ class OpenAIChatSerializer(serializers.Serializer): application_id = self.data.get('application_id') client_id = self.data.get('client_id') client_type = self.data.get('client_type') - chat_id = self.generate_chat(chat_id, application_id, message, client_id) + chat_id = self.generate_chat(chat_id, application_id, message, client_id, + asker=instance.get('form_data', {}).get("asker")) return ChatMessageSerializer( data={ 'chat_id': chat_id, 'message': message, @@ -245,6 +255,7 @@ class OpenAIChatSerializer(serializers.Serializer): 'image_list': instance.get('image_list', []), 'document_list': instance.get('document_list', []), 'audio_list': instance.get('audio_list', []), + 'other_list': instance.get('other_list', []), } ).chat(base_to_response=OpenaiToResponse()) @@ -274,6 +285,7 @@ class ChatMessageSerializer(serializers.Serializer): image_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("picture"))) document_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("document"))) audio_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("Audio"))) + other_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("Other"))) child_node = serializers.DictField(required=False, allow_null=True, error_messages=ErrMessage.dict(_("Child Nodes"))) @@ -372,6 +384,7 @@ class ChatMessageSerializer(serializers.Serializer): image_list = self.data.get('image_list') document_list = self.data.get('document_list') audio_list = self.data.get('audio_list') + other_list = self.data.get('other_list') user_id = chat_info.application.user_id chat_record_id = self.data.get('chat_record_id') chat_record = None @@ -388,7 +401,7 @@ class ChatMessageSerializer(serializers.Serializer): 'client_id': client_id, 'client_type': client_type, 'user_id': user_id}, WorkFlowPostHandler(chat_info, client_id, client_type), - base_to_response, form_data, image_list, document_list, audio_list, + base_to_response, form_data, image_list, document_list, audio_list, other_list, self.data.get('runtime_node_id'), self.data.get('node_data'), chat_record, self.data.get('child_node')) r = work_flow_manage.run() diff --git a/apps/application/views/chat_views.py b/apps/application/views/chat_views.py index 0415f8208..77a087351 100644 --- a/apps/application/views/chat_views.py +++ b/apps/application/views/chat_views.py @@ -144,6 +144,8 @@ class ChatView(APIView): 'document_list') if 'document_list' in request.data else [], 'audio_list': request.data.get( 'audio_list') if 'audio_list' in request.data else [], + 'other_list': request.data.get( + 'other_list') if 'other_list' in request.data else [], 'client_type': request.auth.client_type, 'node_id': request.data.get('node_id', None), 'runtime_node_id': request.data.get('runtime_node_id', None), diff --git a/apps/common/handle/impl/doc_split_handle.py b/apps/common/handle/impl/doc_split_handle.py index 1df7b6a66..4161f13a1 100644 --- a/apps/common/handle/impl/doc_split_handle.py +++ b/apps/common/handle/impl/doc_split_handle.py @@ -112,11 +112,7 @@ def get_image_id_func(): title_font_list = [ [36, 100], - [26, 36], - [24, 26], - [22, 24], - [18, 22], - [16, 18] + [30, 36] ] @@ -130,7 +126,7 @@ def get_title_level(paragraph: Paragraph): if len(paragraph.runs) == 1: font_size = paragraph.runs[0].font.size pt = font_size.pt - if pt >= 16: + if pt >= 30: for _value, index in zip(title_font_list, range(len(title_font_list))): if pt >= _value[0] and pt < _value[1]: return index + 1 diff --git a/apps/locales/zh_CN/LC_MESSAGES/django.po b/apps/locales/zh_CN/LC_MESSAGES/django.po index b0ab7871b..75fec7099 100644 --- a/apps/locales/zh_CN/LC_MESSAGES/django.po +++ b/apps/locales/zh_CN/LC_MESSAGES/django.po @@ -4536,7 +4536,7 @@ msgstr "修改知识库信息" #: community/apps/dataset/views/document.py:463 #: community/apps/dataset/views/document.py:464 msgid "Get the knowledge base paginated list" -msgstr "获取知识库分页列表" +msgstr "获取知识库文档分页列表" #: community/apps/dataset/views/document.py:31 #: community/apps/dataset/views/document.py:32 diff --git a/apps/locales/zh_Hant/LC_MESSAGES/django.po b/apps/locales/zh_Hant/LC_MESSAGES/django.po index dab1d176c..870f282ae 100644 --- a/apps/locales/zh_Hant/LC_MESSAGES/django.po +++ b/apps/locales/zh_Hant/LC_MESSAGES/django.po @@ -4545,7 +4545,7 @@ msgstr "修改知識庫信息" #: community/apps/dataset/views/document.py:463 #: community/apps/dataset/views/document.py:464 msgid "Get the knowledge base paginated list" -msgstr "獲取知識庫分頁列表" +msgstr "獲取知識庫文档分頁列表" #: community/apps/dataset/views/document.py:31 #: community/apps/dataset/views/document.py:32 diff --git a/apps/setting/models_provider/base_model_provider.py b/apps/setting/models_provider/base_model_provider.py index 622be703d..2b02bdc1f 100644 --- a/apps/setting/models_provider/base_model_provider.py +++ b/apps/setting/models_provider/base_model_provider.py @@ -106,7 +106,10 @@ class MaxKBBaseModel(ABC): optional_params = {} for key, value in model_kwargs.items(): if key not in ['model_id', 'use_local', 'streaming', 'show_ref_label']: - optional_params[key] = value + if key == 'extra_body' and isinstance(value, dict): + optional_params = {**optional_params, **value} + else: + optional_params[key] = value return optional_params diff --git a/apps/setting/models_provider/impl/aliyun_bai_lian_model_provider/model/image.py b/apps/setting/models_provider/impl/aliyun_bai_lian_model_provider/model/image.py index 2b1fe31f2..7cda97f23 100644 --- a/apps/setting/models_provider/impl/aliyun_bai_lian_model_provider/model/image.py +++ b/apps/setting/models_provider/impl/aliyun_bai_lian_model_provider/model/image.py @@ -15,9 +15,8 @@ class QwenVLChatModel(MaxKBBaseModel, BaseChatOpenAI): model_name=model_name, openai_api_key=model_credential.get('api_key'), openai_api_base='https://dashscope.aliyuncs.com/compatible-mode/v1', - # stream_options={"include_usage": True}, streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) return chat_tong_yi diff --git a/apps/setting/models_provider/impl/aliyun_bai_lian_model_provider/model/llm.py b/apps/setting/models_provider/impl/aliyun_bai_lian_model_provider/model/llm.py index d914f7c8a..ee3ee6488 100644 --- a/apps/setting/models_provider/impl/aliyun_bai_lian_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/aliyun_bai_lian_model_provider/model/llm.py @@ -20,5 +20,5 @@ class BaiLianChatModel(MaxKBBaseModel, BaseChatOpenAI): model=model_name, openai_api_base=model_credential.get('api_base'), openai_api_key=model_credential.get('api_key'), - **optional_params + extra_body=optional_params ) diff --git a/apps/setting/models_provider/impl/aws_bedrock_model_provider/model/llm.py b/apps/setting/models_provider/impl/aws_bedrock_model_provider/model/llm.py index ef1c13337..7b0088a4a 100644 --- a/apps/setting/models_provider/impl/aws_bedrock_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/aws_bedrock_model_provider/model/llm.py @@ -1,10 +1,12 @@ import os import re -from typing import Dict +from typing import Dict, List from botocore.config import Config from langchain_community.chat_models import BedrockChat +from langchain_core.messages import BaseMessage, get_buffer_string +from common.config.tokenizer_manage_config import TokenizerManage from setting.models_provider.base_model_provider import MaxKBBaseModel @@ -72,6 +74,20 @@ class BedrockModel(MaxKBBaseModel, BedrockChat): config=config ) + def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int: + try: + return super().get_num_tokens_from_messages(messages) + except Exception as e: + tokenizer = TokenizerManage.get_tokenizer() + return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages]) + + def get_num_tokens(self, text: str) -> int: + try: + return super().get_num_tokens(text) + except Exception as e: + tokenizer = TokenizerManage.get_tokenizer() + return len(tokenizer.encode(text)) + def _update_aws_credentials(profile_name, access_key_id, secret_access_key): credentials_path = os.path.join(os.path.expanduser("~"), ".aws", "credentials") diff --git a/apps/setting/models_provider/impl/base_chat_open_ai.py b/apps/setting/models_provider/impl/base_chat_open_ai.py index 54076b7ef..c96bfacf2 100644 --- a/apps/setting/models_provider/impl/base_chat_open_ai.py +++ b/apps/setting/models_provider/impl/base_chat_open_ai.py @@ -1,15 +1,16 @@ # coding=utf-8 -import warnings -from typing import List, Dict, Optional, Any, Iterator, cast, Type, Union +from typing import Dict, Optional, Any, Iterator, cast, Union, Sequence, Callable, Mapping -import openai -from langchain_core.callbacks import CallbackManagerForLLMRun from langchain_core.language_models import LanguageModelInput -from langchain_core.messages import BaseMessage, get_buffer_string, BaseMessageChunk, AIMessageChunk -from langchain_core.outputs import ChatGenerationChunk, ChatGeneration +from langchain_core.messages import BaseMessage, get_buffer_string, BaseMessageChunk, HumanMessageChunk, AIMessageChunk, \ + SystemMessageChunk, FunctionMessageChunk, ChatMessageChunk +from langchain_core.messages.ai import UsageMetadata +from langchain_core.messages.tool import tool_call_chunk, ToolMessageChunk +from langchain_core.outputs import ChatGenerationChunk from langchain_core.runnables import RunnableConfig, ensure_config -from langchain_core.utils.pydantic import is_basemodel_subclass +from langchain_core.tools import BaseTool from langchain_openai import ChatOpenAI +from langchain_openai.chat_models.base import _create_usage_metadata from common.config.tokenizer_manage_config import TokenizerManage @@ -19,6 +20,65 @@ def custom_get_token_ids(text: str): return tokenizer.encode(text) +def _convert_delta_to_message_chunk( + _dict: Mapping[str, Any], default_class: type[BaseMessageChunk] +) -> BaseMessageChunk: + id_ = _dict.get("id") + role = cast(str, _dict.get("role")) + content = cast(str, _dict.get("content") or "") + additional_kwargs: dict = {} + if 'reasoning_content' in _dict: + additional_kwargs['reasoning_content'] = _dict.get('reasoning_content') + if _dict.get("function_call"): + function_call = dict(_dict["function_call"]) + if "name" in function_call and function_call["name"] is None: + function_call["name"] = "" + additional_kwargs["function_call"] = function_call + tool_call_chunks = [] + if raw_tool_calls := _dict.get("tool_calls"): + additional_kwargs["tool_calls"] = raw_tool_calls + try: + tool_call_chunks = [ + tool_call_chunk( + name=rtc["function"].get("name"), + args=rtc["function"].get("arguments"), + id=rtc.get("id"), + index=rtc["index"], + ) + for rtc in raw_tool_calls + ] + except KeyError: + pass + + if role == "user" or default_class == HumanMessageChunk: + return HumanMessageChunk(content=content, id=id_) + elif role == "assistant" or default_class == AIMessageChunk: + return AIMessageChunk( + content=content, + additional_kwargs=additional_kwargs, + id=id_, + tool_call_chunks=tool_call_chunks, # type: ignore[arg-type] + ) + elif role in ("system", "developer") or default_class == SystemMessageChunk: + if role == "developer": + additional_kwargs = {"__openai_role__": "developer"} + else: + additional_kwargs = {} + return SystemMessageChunk( + content=content, id=id_, additional_kwargs=additional_kwargs + ) + elif role == "function" or default_class == FunctionMessageChunk: + return FunctionMessageChunk(content=content, name=_dict["name"], id=id_) + elif role == "tool" or default_class == ToolMessageChunk: + return ToolMessageChunk( + content=content, tool_call_id=_dict["tool_call_id"], id=id_ + ) + elif role or default_class == ChatMessageChunk: + return ChatMessageChunk(content=content, role=role, id=id_) + else: + return default_class(content=content, id=id_) # type: ignore + + class BaseChatOpenAI(ChatOpenAI): usage_metadata: dict = {} custom_get_token_ids = custom_get_token_ids @@ -26,7 +86,13 @@ class BaseChatOpenAI(ChatOpenAI): def get_last_generation_info(self) -> Optional[Dict[str, Any]]: return self.usage_metadata - def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int: + def get_num_tokens_from_messages( + self, + messages: list[BaseMessage], + tools: Optional[ + Sequence[Union[dict[str, Any], type, Callable, BaseTool]] + ] = None, + ) -> int: if self.usage_metadata is None or self.usage_metadata == {}: try: return super().get_num_tokens_from_messages(messages) @@ -44,114 +110,77 @@ class BaseChatOpenAI(ChatOpenAI): return len(tokenizer.encode(text)) return self.get_last_generation_info().get('output_tokens', 0) - def _stream( + def _stream(self, *args: Any, **kwargs: Any) -> Iterator[ChatGenerationChunk]: + kwargs['stream_usage'] = True + for chunk in super()._stream(*args, **kwargs): + if chunk.message.usage_metadata is not None: + self.usage_metadata = chunk.message.usage_metadata + yield chunk + + def _convert_chunk_to_generation_chunk( self, - messages: List[BaseMessage], - stop: Optional[List[str]] = None, - run_manager: Optional[CallbackManagerForLLMRun] = None, - **kwargs: Any, - ) -> Iterator[ChatGenerationChunk]: - kwargs["stream"] = True - kwargs["stream_options"] = {"include_usage": True} - """Set default stream_options.""" - stream_usage = self._should_stream_usage(kwargs.get('stream_usage'), **kwargs) - # Note: stream_options is not a valid parameter for Azure OpenAI. - # To support users proxying Azure through ChatOpenAI, here we only specify - # stream_options if include_usage is set to True. - # See https://learn.microsoft.com/en-us/azure/ai-services/openai/whats-new - # for release notes. - if stream_usage: - kwargs["stream_options"] = {"include_usage": stream_usage} + chunk: dict, + default_chunk_class: type, + base_generation_info: Optional[dict], + ) -> Optional[ChatGenerationChunk]: + if chunk.get("type") == "content.delta": # from beta.chat.completions.stream + return None + token_usage = chunk.get("usage") + choices = ( + chunk.get("choices", []) + # from beta.chat.completions.stream + or chunk.get("chunk", {}).get("choices", []) + ) - payload = self._get_request_payload(messages, stop=stop, **kwargs) - default_chunk_class: Type[BaseMessageChunk] = AIMessageChunk - base_generation_info = {} - - if "response_format" in payload and is_basemodel_subclass( - payload["response_format"] - ): - # TODO: Add support for streaming with Pydantic response_format. - warnings.warn("Streaming with Pydantic response_format not yet supported.") - chat_result = self._generate( - messages, stop, run_manager=run_manager, **kwargs + usage_metadata: Optional[UsageMetadata] = ( + _create_usage_metadata(token_usage) if token_usage else None + ) + if len(choices) == 0: + # logprobs is implicitly None + generation_chunk = ChatGenerationChunk( + message=default_chunk_class(content="", usage_metadata=usage_metadata) ) - msg = chat_result.generations[0].message - yield ChatGenerationChunk( - message=AIMessageChunk( - **msg.dict(exclude={"type", "additional_kwargs"}), - # preserve the "parsed" Pydantic object without converting to dict - additional_kwargs=msg.additional_kwargs, - ), - generation_info=chat_result.generations[0].generation_info, - ) - return - if self.include_response_headers: - raw_response = self.client.with_raw_response.create(**payload) - response = raw_response.parse() - base_generation_info = {"headers": dict(raw_response.headers)} - else: - response = self.client.create(**payload) - with response: - is_first_chunk = True - for chunk in response: - if not isinstance(chunk, dict): - chunk = chunk.model_dump() + return generation_chunk - generation_chunk = super()._convert_chunk_to_generation_chunk( - chunk, - default_chunk_class, - base_generation_info if is_first_chunk else {}, - ) - if generation_chunk is None: - continue + choice = choices[0] + if choice["delta"] is None: + return None - # custom code - if len(chunk['choices']) > 0 and 'reasoning_content' in chunk['choices'][0]['delta']: - generation_chunk.message.additional_kwargs["reasoning_content"] = chunk['choices'][0]['delta'][ - 'reasoning_content'] + message_chunk = _convert_delta_to_message_chunk( + choice["delta"], default_chunk_class + ) + generation_info = {**base_generation_info} if base_generation_info else {} - default_chunk_class = generation_chunk.message.__class__ - logprobs = (generation_chunk.generation_info or {}).get("logprobs") - if run_manager: - run_manager.on_llm_new_token( - generation_chunk.text, chunk=generation_chunk, logprobs=logprobs - ) - is_first_chunk = False - # custom code - if generation_chunk.message.usage_metadata is not None: - self.usage_metadata = generation_chunk.message.usage_metadata - yield generation_chunk + if finish_reason := choice.get("finish_reason"): + generation_info["finish_reason"] = finish_reason + if model_name := chunk.get("model"): + generation_info["model_name"] = model_name + if system_fingerprint := chunk.get("system_fingerprint"): + generation_info["system_fingerprint"] = system_fingerprint - def _create_chat_result(self, - response: Union[dict, openai.BaseModel], - generation_info: Optional[Dict] = None): - result = super()._create_chat_result(response, generation_info) - try: - reasoning_content = '' - reasoning_content_enable = False - for res in response.choices: - if 'reasoning_content' in res.message.model_extra: - reasoning_content_enable = True - _reasoning_content = res.message.model_extra.get('reasoning_content') - if _reasoning_content is not None: - reasoning_content += _reasoning_content - if reasoning_content_enable: - result.llm_output['reasoning_content'] = reasoning_content - except Exception as e: - pass - return result + logprobs = choice.get("logprobs") + if logprobs: + generation_info["logprobs"] = logprobs + + if usage_metadata and isinstance(message_chunk, AIMessageChunk): + message_chunk.usage_metadata = usage_metadata + + generation_chunk = ChatGenerationChunk( + message=message_chunk, generation_info=generation_info or None + ) + return generation_chunk def invoke( self, input: LanguageModelInput, config: Optional[RunnableConfig] = None, *, - stop: Optional[List[str]] = None, + stop: Optional[list[str]] = None, **kwargs: Any, ) -> BaseMessage: config = ensure_config(config) chat_result = cast( - ChatGeneration, + "ChatGeneration", self.generate_prompt( [self._convert_input(input)], stop=stop, @@ -162,7 +191,9 @@ class BaseChatOpenAI(ChatOpenAI): run_id=config.pop("run_id", None), **kwargs, ).generations[0][0], + ).message + self.usage_metadata = chat_result.response_metadata[ 'token_usage'] if 'token_usage' in chat_result.response_metadata else chat_result.usage_metadata return chat_result diff --git a/apps/setting/models_provider/impl/deepseek_model_provider/model/llm.py b/apps/setting/models_provider/impl/deepseek_model_provider/model/llm.py index 9db4faca7..081d648a7 100644 --- a/apps/setting/models_provider/impl/deepseek_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/deepseek_model_provider/model/llm.py @@ -26,6 +26,6 @@ class DeepSeekChatModel(MaxKBBaseModel, BaseChatOpenAI): model=model_name, openai_api_base='https://api.deepseek.com', openai_api_key=model_credential.get('api_key'), - **optional_params + extra_body=optional_params ) return deepseek_chat_open_ai diff --git a/apps/setting/models_provider/impl/kimi_model_provider/model/llm.py b/apps/setting/models_provider/impl/kimi_model_provider/model/llm.py index c389c177e..c0ce2ec02 100644 --- a/apps/setting/models_provider/impl/kimi_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/kimi_model_provider/model/llm.py @@ -21,11 +21,10 @@ class KimiChatModel(MaxKBBaseModel, BaseChatOpenAI): @staticmethod def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs): optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs) - kimi_chat_open_ai = KimiChatModel( openai_api_base=model_credential['api_base'], openai_api_key=model_credential['api_key'], model_name=model_name, - **optional_params + extra_body=optional_params, ) return kimi_chat_open_ai diff --git a/apps/setting/models_provider/impl/ollama_model_provider/model/image.py b/apps/setting/models_provider/impl/ollama_model_provider/model/image.py index 4cf0f1d56..215ce0130 100644 --- a/apps/setting/models_provider/impl/ollama_model_provider/model/image.py +++ b/apps/setting/models_provider/impl/ollama_model_provider/model/image.py @@ -28,5 +28,5 @@ class OllamaImage(MaxKBBaseModel, BaseChatOpenAI): # stream_options={"include_usage": True}, streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) diff --git a/apps/setting/models_provider/impl/openai_model_provider/model/image.py b/apps/setting/models_provider/impl/openai_model_provider/model/image.py index 731f476c4..7ac0906a7 100644 --- a/apps/setting/models_provider/impl/openai_model_provider/model/image.py +++ b/apps/setting/models_provider/impl/openai_model_provider/model/image.py @@ -16,5 +16,5 @@ class OpenAIImage(MaxKBBaseModel, BaseChatOpenAI): # stream_options={"include_usage": True}, streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) diff --git a/apps/setting/models_provider/impl/openai_model_provider/model/llm.py b/apps/setting/models_provider/impl/openai_model_provider/model/llm.py index 2e6dd89ac..189385210 100644 --- a/apps/setting/models_provider/impl/openai_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/openai_model_provider/model/llm.py @@ -9,7 +9,6 @@ from typing import List, Dict from langchain_core.messages import BaseMessage, get_buffer_string -from langchain_openai.chat_models import ChatOpenAI from common.config.tokenizer_manage_config import TokenizerManage from setting.models_provider.base_model_provider import MaxKBBaseModel @@ -35,9 +34,9 @@ class OpenAIChatModel(MaxKBBaseModel, BaseChatOpenAI): streaming = False azure_chat_open_ai = OpenAIChatModel( model=model_name, - openai_api_base=model_credential.get('api_base'), - openai_api_key=model_credential.get('api_key'), - **optional_params, + base_url=model_credential.get('api_base'), + api_key=model_credential.get('api_key'), + extra_body=optional_params, streaming=streaming, custom_get_token_ids=custom_get_token_ids ) diff --git a/apps/setting/models_provider/impl/qwen_model_provider/model/image.py b/apps/setting/models_provider/impl/qwen_model_provider/model/image.py index 97166757e..bf3af0e34 100644 --- a/apps/setting/models_provider/impl/qwen_model_provider/model/image.py +++ b/apps/setting/models_provider/impl/qwen_model_provider/model/image.py @@ -18,9 +18,8 @@ class QwenVLChatModel(MaxKBBaseModel, BaseChatOpenAI): model_name=model_name, openai_api_key=model_credential.get('api_key'), openai_api_base='https://dashscope.aliyuncs.com/compatible-mode/v1', - # stream_options={"include_usage": True}, streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) return chat_tong_yi diff --git a/apps/setting/models_provider/impl/qwen_model_provider/model/llm.py b/apps/setting/models_provider/impl/qwen_model_provider/model/llm.py index 3b66ddfd6..c4df28af9 100644 --- a/apps/setting/models_provider/impl/qwen_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/qwen_model_provider/model/llm.py @@ -26,6 +26,6 @@ class QwenChatModel(MaxKBBaseModel, BaseChatOpenAI): openai_api_base='https://dashscope.aliyuncs.com/compatible-mode/v1', streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) return chat_tong_yi diff --git a/apps/setting/models_provider/impl/siliconCloud_model_provider/model/image.py b/apps/setting/models_provider/impl/siliconCloud_model_provider/model/image.py index bb840f8c6..2ec0689d4 100644 --- a/apps/setting/models_provider/impl/siliconCloud_model_provider/model/image.py +++ b/apps/setting/models_provider/impl/siliconCloud_model_provider/model/image.py @@ -16,5 +16,5 @@ class SiliconCloudImage(MaxKBBaseModel, BaseChatOpenAI): # stream_options={"include_usage": True}, streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) diff --git a/apps/setting/models_provider/impl/siliconCloud_model_provider/model/llm.py b/apps/setting/models_provider/impl/siliconCloud_model_provider/model/llm.py index 9d79c6e07..6fb0c7816 100644 --- a/apps/setting/models_provider/impl/siliconCloud_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/siliconCloud_model_provider/model/llm.py @@ -34,5 +34,5 @@ class SiliconCloudChatModel(MaxKBBaseModel, BaseChatOpenAI): model=model_name, openai_api_base=model_credential.get('api_base'), openai_api_key=model_credential.get('api_key'), - **optional_params + extra_body=optional_params ) diff --git a/apps/setting/models_provider/impl/tencent_cloud_model_provider/model/llm.py b/apps/setting/models_provider/impl/tencent_cloud_model_provider/model/llm.py index 7653cfc2f..cfcdf7aca 100644 --- a/apps/setting/models_provider/impl/tencent_cloud_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/tencent_cloud_model_provider/model/llm.py @@ -33,21 +33,7 @@ class TencentCloudChatModel(MaxKBBaseModel, BaseChatOpenAI): model=model_name, openai_api_base=model_credential.get('api_base'), openai_api_key=model_credential.get('api_key'), - **optional_params, + extra_body=optional_params, custom_get_token_ids=custom_get_token_ids ) return azure_chat_open_ai - - def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int: - try: - return super().get_num_tokens_from_messages(messages) - except Exception as e: - tokenizer = TokenizerManage.get_tokenizer() - return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages]) - - def get_num_tokens(self, text: str) -> int: - try: - return super().get_num_tokens(text) - except Exception as e: - tokenizer = TokenizerManage.get_tokenizer() - return len(tokenizer.encode(text)) diff --git a/apps/setting/models_provider/impl/tencent_model_provider/model/image.py b/apps/setting/models_provider/impl/tencent_model_provider/model/image.py index 1b66ab6d2..6800cdd56 100644 --- a/apps/setting/models_provider/impl/tencent_model_provider/model/image.py +++ b/apps/setting/models_provider/impl/tencent_model_provider/model/image.py @@ -16,5 +16,5 @@ class TencentVision(MaxKBBaseModel, BaseChatOpenAI): # stream_options={"include_usage": True}, streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) diff --git a/apps/setting/models_provider/impl/vllm_model_provider/model/image.py b/apps/setting/models_provider/impl/vllm_model_provider/model/image.py index 4d5dda29d..c8cb0a84d 100644 --- a/apps/setting/models_provider/impl/vllm_model_provider/model/image.py +++ b/apps/setting/models_provider/impl/vllm_model_provider/model/image.py @@ -19,7 +19,7 @@ class VllmImage(MaxKBBaseModel, BaseChatOpenAI): # stream_options={"include_usage": True}, streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) def is_cache_model(self): diff --git a/apps/setting/models_provider/impl/vllm_model_provider/model/llm.py b/apps/setting/models_provider/impl/vllm_model_provider/model/llm.py index 7d2a63acd..4662a6169 100644 --- a/apps/setting/models_provider/impl/vllm_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/vllm_model_provider/model/llm.py @@ -1,9 +1,10 @@ # coding=utf-8 -from typing import Dict, List +from typing import Dict, Optional, Sequence, Union, Any, Callable from urllib.parse import urlparse, ParseResult from langchain_core.messages import BaseMessage, get_buffer_string +from langchain_core.tools import BaseTool from common.config.tokenizer_manage_config import TokenizerManage from setting.models_provider.base_model_provider import MaxKBBaseModel @@ -31,13 +32,19 @@ class VllmChatModel(MaxKBBaseModel, BaseChatOpenAI): model=model_name, openai_api_base=model_credential.get('api_base'), openai_api_key=model_credential.get('api_key'), - **optional_params, streaming=True, stream_usage=True, + extra_body=optional_params ) return vllm_chat_open_ai - def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int: + def get_num_tokens_from_messages( + self, + messages: list[BaseMessage], + tools: Optional[ + Sequence[Union[dict[str, Any], type, Callable, BaseTool]] + ] = None, + ) -> int: if self.usage_metadata is None or self.usage_metadata == {}: tokenizer = TokenizerManage.get_tokenizer() return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages]) diff --git a/apps/setting/models_provider/impl/volcanic_engine_model_provider/model/image.py b/apps/setting/models_provider/impl/volcanic_engine_model_provider/model/image.py index 39446b4e1..6e2517bd4 100644 --- a/apps/setting/models_provider/impl/volcanic_engine_model_provider/model/image.py +++ b/apps/setting/models_provider/impl/volcanic_engine_model_provider/model/image.py @@ -16,5 +16,5 @@ class VolcanicEngineImage(MaxKBBaseModel, BaseChatOpenAI): # stream_options={"include_usage": True}, streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) diff --git a/apps/setting/models_provider/impl/volcanic_engine_model_provider/model/llm.py b/apps/setting/models_provider/impl/volcanic_engine_model_provider/model/llm.py index 181ad2971..8f089f269 100644 --- a/apps/setting/models_provider/impl/volcanic_engine_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/volcanic_engine_model_provider/model/llm.py @@ -17,5 +17,5 @@ class VolcanicEngineChatModel(MaxKBBaseModel, BaseChatOpenAI): model=model_name, openai_api_base=model_credential.get('api_base'), openai_api_key=model_credential.get('api_key'), - **optional_params + extra_body=optional_params ) diff --git a/apps/setting/models_provider/impl/wenxin_model_provider/credential/llm.py b/apps/setting/models_provider/impl/wenxin_model_provider/credential/llm.py index 06ec94aae..d4d379db3 100644 --- a/apps/setting/models_provider/impl/wenxin_model_provider/credential/llm.py +++ b/apps/setting/models_provider/impl/wenxin_model_provider/credential/llm.py @@ -27,7 +27,7 @@ class WenxinLLMModelParams(BaseForm): _step=0.01, precision=2) - max_tokens = forms.SliderField( + max_output_tokens = forms.SliderField( TooltipLabel(_('Output the maximum Tokens'), _('Specify the maximum number of tokens that the model can generate')), required=True, default_value=1024, diff --git a/apps/setting/models_provider/impl/xinference_model_provider/model/image.py b/apps/setting/models_provider/impl/xinference_model_provider/model/image.py index a195b8649..66a766ba8 100644 --- a/apps/setting/models_provider/impl/xinference_model_provider/model/image.py +++ b/apps/setting/models_provider/impl/xinference_model_provider/model/image.py @@ -19,7 +19,7 @@ class XinferenceImage(MaxKBBaseModel, BaseChatOpenAI): # stream_options={"include_usage": True}, streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int: diff --git a/apps/setting/models_provider/impl/xinference_model_provider/model/llm.py b/apps/setting/models_provider/impl/xinference_model_provider/model/llm.py index d76979bd3..9c0316ad2 100644 --- a/apps/setting/models_provider/impl/xinference_model_provider/model/llm.py +++ b/apps/setting/models_provider/impl/xinference_model_provider/model/llm.py @@ -34,7 +34,7 @@ class XinferenceChatModel(MaxKBBaseModel, BaseChatOpenAI): model=model_name, openai_api_base=base_url, openai_api_key=model_credential.get('api_key'), - **optional_params + extra_body=optional_params ) def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int: diff --git a/apps/setting/models_provider/impl/zhipu_model_provider/model/image.py b/apps/setting/models_provider/impl/zhipu_model_provider/model/image.py index f13c71538..6ac7830d8 100644 --- a/apps/setting/models_provider/impl/zhipu_model_provider/model/image.py +++ b/apps/setting/models_provider/impl/zhipu_model_provider/model/image.py @@ -16,5 +16,5 @@ class ZhiPuImage(MaxKBBaseModel, BaseChatOpenAI): # stream_options={"include_usage": True}, streaming=True, stream_usage=True, - **optional_params, + extra_body=optional_params ) diff --git a/pyproject.toml b/pyproject.toml index 35d74a52e..b9e66a61e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ package-mode = false [tool.poetry.dependencies] python = ">=3.11,<3.12" -django = "4.2.18" +django = "4.2.20" djangorestframework = "^3.15.2" drf-yasg = "1.21.7" django-filter = "23.2" diff --git a/ui/src/api/type/application.ts b/ui/src/api/type/application.ts index 077e23097..c423f1110 100644 --- a/ui/src/api/type/application.ts +++ b/ui/src/api/type/application.ts @@ -72,6 +72,7 @@ interface chatType { document_list: Array image_list: Array audio_list: Array + other_list: Array } } diff --git a/ui/src/components/ai-chat/ExecutionDetailDialog.vue b/ui/src/components/ai-chat/ExecutionDetailDialog.vue index 0f2296439..98ec1e6fa 100644 --- a/ui/src/components/ai-chat/ExecutionDetailDialog.vue +++ b/ui/src/components/ai-chat/ExecutionDetailDialog.vue @@ -125,6 +125,28 @@ +
+

+ {{ $t('common.fileUpload.document') }}: +

+ + + + +
diff --git a/ui/src/components/ai-chat/component/answer-content/index.vue b/ui/src/components/ai-chat/component/answer-content/index.vue index 7f09fa04c..26cd8a0d0 100644 --- a/ui/src/components/ai-chat/component/answer-content/index.vue +++ b/ui/src/components/ai-chat/component/answer-content/index.vue @@ -80,7 +80,7 @@ const props = defineProps<{ chatRecord: chatType application: any loading: boolean - sendMessage: (question: string, other_params_data?: any, chat?: chatType) => void + sendMessage: (question: string, other_params_data?: any, chat?: chatType) => Promise chatManagement: any type: 'log' | 'ai-chat' | 'debug-ai-chat' }>() @@ -98,9 +98,10 @@ const showUserAvatar = computed(() => { const chatMessage = (question: string, type: 'old' | 'new', other_params_data?: any) => { if (type === 'old') { add_answer_text_list(props.chatRecord.answer_text_list) - props.sendMessage(question, other_params_data, props.chatRecord) - props.chatManagement.open(props.chatRecord.id) - props.chatManagement.write(props.chatRecord.id) + props.sendMessage(question, other_params_data, props.chatRecord).then(() => { + props.chatManagement.open(props.chatRecord.id) + props.chatManagement.write(props.chatRecord.id) + }) } else { props.sendMessage(question, other_params_data) } diff --git a/ui/src/components/ai-chat/component/chat-input-operate/index.vue b/ui/src/components/ai-chat/component/chat-input-operate/index.vue index acf3085ed..ad6a86ff2 100644 --- a/ui/src/components/ai-chat/component/chat-input-operate/index.vue +++ b/ui/src/components/ai-chat/component/chat-input-operate/index.vue @@ -10,7 +10,8 @@ uploadDocumentList.length || uploadImageList.length || uploadAudioList.length || - uploadVideoList.length + uploadVideoList.length || + uploadOtherList.length " > @@ -30,22 +31,62 @@ class="file cursor" >
+
+ +
+ {{ item && item?.name }} +
+
- +
- -
- {{ item && item?.name }} +
+ + + + +
+
+ +
+ {{ item && item?.name }} +
+
+
+ + +
@@ -63,23 +104,25 @@ >
+
+ +
+ {{ item && item?.name }} +
+
- +
- -
- {{ item && item?.name }} -
@@ -87,7 +130,7 @@