feat: add speech_to_text node and text_to_speech node

apps/application/flow/step_node/__init__.py

@@ -21,12 +21,14 @@ from .image_understand_step_node import *
 from .image_generate_step_node import *
 
 from .search_dataset_node import *
+from .speech_to_text_step_node import BaseSpeechToTextNode
 from .start_node import *
+from .text_to_speech_step_node.impl.base_text_to_speech_node import BaseTextToSpeechNode
 
 node_list = [BaseStartStepNode, BaseChatNode, BaseSearchDatasetNode, BaseQuestionNode, BaseConditionNode, BaseReplyNode,
              BaseFunctionNodeNode, BaseFunctionLibNodeNode, BaseRerankerNode, BaseApplicationNode,
              BaseDocumentExtractNode,
-             BaseImageUnderstandNode, BaseImageGenerateNode, BaseFormNode]
+             BaseImageUnderstandNode, BaseFormNode, BaseSpeechToTextNode, BaseTextToSpeechNode,BaseImageGenerateNode]
 
 
 def get_node(node_type):

apps/application/flow/step_node/application_node/i_application_node.py

@@ -14,6 +14,7 @@ class ApplicationNodeSerializer(serializers.Serializer):
     user_input_field_list = serializers.ListField(required=False, error_messages=ErrMessage.uuid("用户输入字段"))
     image_list = serializers.ListField(required=False, error_messages=ErrMessage.list("图片"))
     document_list = serializers.ListField(required=False, error_messages=ErrMessage.list("文档"))
+    audio_list = serializers.ListField(required=False, error_messages=ErrMessage.list("音频"))
     child_node = serializers.DictField(required=False, allow_null=True, error_messages=ErrMessage.dict("子节点"))
     node_data = serializers.DictField(required=False, allow_null=True, error_messages=ErrMessage.dict("表单数据"))
 
@@ -43,7 +44,7 @@ class IApplicationNode(INode):
                 app_document_list[1:])
             for document in app_document_list:
                 if 'file_id' not in document:
-                    raise ValueError("参数值错误: 上传的文档中缺少file_id")
+                    raise ValueError("参数值错误: 上传的文档中缺少file_id，文档上传失败")
         app_image_list = self.node_params_serializer.data.get('image_list', [])
         if app_image_list and len(app_image_list) > 0:
             app_image_list = self.workflow_manage.get_reference_field(
@@ -51,11 +52,22 @@ class IApplicationNode(INode):
                 app_image_list[1:])
             for image in app_image_list:
                 if 'file_id' not in image:
-                    raise ValueError("参数值错误: 上传的图片中缺少file_id")
+                    raise ValueError("参数值错误: 上传的图片中缺少file_id，图片上传失败")
+
+        app_audio_list = self.node_params_serializer.data.get('audio_list', [])
+        if app_audio_list and len(app_audio_list) > 0:
+            app_audio_list = self.workflow_manage.get_reference_field(
+                app_audio_list[0],
+                app_audio_list[1:])
+            for audio in app_audio_list:
+                if 'file_id' not in audio:
+                    raise ValueError("参数值错误: 上传的图片中缺少file_id，音频上传失败")
         return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data,
                             app_document_list=app_document_list, app_image_list=app_image_list,
+                            app_audio_list=app_audio_list,
                             message=str(question), **kwargs)
 
     def execute(self, application_id, message, chat_id, chat_record_id, stream, re_chat, client_id, client_type,
-                app_document_list=None, app_image_list=None, child_node=None, node_data=None, **kwargs) -> NodeResult:
+                app_document_list=None, app_image_list=None, app_audio_list=None, child_node=None, node_data=None,
+                **kwargs) -> NodeResult:
         pass

apps/application/flow/step_node/application_node/impl/base_application_node.py

@@ -154,7 +154,7 @@ class BaseApplicationNode(IApplicationNode):
         self.answer_text = details.get('answer')
 
     def execute(self, application_id, message, chat_id, chat_record_id, stream, re_chat, client_id, client_type,
-                app_document_list=None, app_image_list=None, child_node=None, node_data=None,
+                app_document_list=None, app_image_list=None, app_audio_list=None, child_node=None, node_data=None,
                 **kwargs) -> NodeResult:
         from application.serializers.chat_message_serializers import ChatMessageSerializer
         # 生成嵌入应用的chat_id
@@ -167,6 +167,8 @@ class BaseApplicationNode(IApplicationNode):
             app_document_list = []
         if app_image_list is None:
             app_image_list = []
+        if app_audio_list is None:
+            app_audio_list = []
         runtime_node_id = None
         record_id = None
         child_node_value = None
@@ -186,6 +188,7 @@ class BaseApplicationNode(IApplicationNode):
                   'client_type': client_type,
                   'document_list': app_document_list,
                   'image_list': app_image_list,
+                  'audio_list': app_audio_list,
                   'runtime_node_id': runtime_node_id,
                   'chat_record_id': record_id,
                   'child_node': child_node_value,
@@ -234,5 +237,6 @@ class BaseApplicationNode(IApplicationNode):
             'global_fields': global_fields,
             'document_list': self.workflow_manage.document_list,
             'image_list': self.workflow_manage.image_list,
+            'audio_list': self.workflow_manage.audio_list,
             'application_node_dict': self.context.get('application_node_dict')
         }

apps/application/flow/step_node/speech_to_text_step_node/__init__.py

@@ -0,0 +1,3 @@
+# coding=utf-8
+
+from .impl import *

apps/application/flow/step_node/speech_to_text_step_node/i_speech_to_text_node.py

@@ -0,0 +1,37 @@
+# coding=utf-8
+
+from typing import Type
+
+from rest_framework import serializers
+
+from application.flow.i_step_node import INode, NodeResult
+from common.util.field_message import ErrMessage
+
+
+class SpeechToTextNodeSerializer(serializers.Serializer):
+    stt_model_id = serializers.CharField(required=True, error_messages=ErrMessage.char("模型id"))
+
+    is_result = serializers.BooleanField(required=False, error_messages=ErrMessage.boolean('是否返回内容'))
+
+    audio_list = serializers.ListField(required=False, error_messages=ErrMessage.list("音频"))
+
+
+class ISpeechToTextNode(INode):
+    type = 'speech-to-text-node'
+
+    def get_node_params_serializer_class(self) -> Type[serializers.Serializer]:
+        return SpeechToTextNodeSerializer
+
+    def _run(self):
+        res = self.workflow_manage.get_reference_field(self.node_params_serializer.data.get('audio_list')[0],
+                                                       self.node_params_serializer.data.get('audio_list')[1:])
+        for audio in res:
+            if 'file_id' not in audio:
+                raise ValueError("参数值错误: 上传的图片中缺少file_id，音频上传失败")
+
+        return self.execute(audio=res, **self.node_params_serializer.data, **self.flow_params_serializer.data)
+
+    def execute(self, stt_model_id, chat_id,
+                audio,
+                **kwargs) -> NodeResult:
+        pass

apps/application/flow/step_node/speech_to_text_step_node/impl/__init__.py

@@ -0,0 +1,3 @@
+# coding=utf-8
+
+from .base_speech_to_text_node import BaseSpeechToTextNode

apps/application/flow/step_node/speech_to_text_step_node/impl/base_speech_to_text_node.py

@@ -0,0 +1,58 @@
+# coding=utf-8
+import os
+import tempfile
+import time
+import io
+from typing import List, Dict
+
+from django.db.models import QuerySet
+from pydub import AudioSegment
+from concurrent.futures import ThreadPoolExecutor
+from application.flow.i_step_node import NodeResult, INode
+from application.flow.step_node.speech_to_text_step_node.i_speech_to_text_node import ISpeechToTextNode
+from common.util.common import split_and_transcribe
+from dataset.models import File
+from setting.models_provider.tools import get_model_instance_by_model_user_id
+
+
+class BaseSpeechToTextNode(ISpeechToTextNode):
+
+    def save_context(self, details, workflow_manage):
+        self.context['answer'] = details.get('answer')
+        self.answer_text = details.get('answer')
+
+    def execute(self, stt_model_id, chat_id, audio, **kwargs) -> NodeResult:
+        stt_model = get_model_instance_by_model_user_id(stt_model_id, self.flow_params_serializer.data.get('user_id'))
+        audio_list = audio
+        self.context['audio_list'] = audio
+
+
+        def process_audio_item(audio_item, model):
+            file = QuerySet(File).filter(id=audio_item['file_id']).first()
+            with tempfile.NamedTemporaryFile(delete=False, suffix='.mp3') as temp_file:
+                temp_file.write(file.get_byte().tobytes())
+                temp_file_path = temp_file.name
+            try:
+                return split_and_transcribe(temp_file_path, model)
+            finally:
+                os.remove(temp_file_path)
+
+        def process_audio_items(audio_list, model):
+            with ThreadPoolExecutor(max_workers=5) as executor:
+                results = list(executor.map(lambda item: process_audio_item(item, model), audio_list))
+            return '\n\n'.join(results)
+
+        result = process_audio_items(audio_list, stt_model)
+        return NodeResult({'answer': result, 'result': result}, {})
+
+    def get_details(self, index: int, **kwargs):
+        return {
+            'name': self.node.properties.get('stepName'),
+            "index": index,
+            'run_time': self.context.get('run_time'),
+            'answer': self.context.get('answer'),
+            'type': self.node.type,
+            'status': self.status,
+            'err_message': self.err_message,
+            'audio_list': self.context.get('audio_list'),
+        }

apps/application/flow/step_node/start_node/impl/base_start_node.py

@@ -39,6 +39,7 @@ class BaseStartStepNode(IStarNode):
         self.context['run_time'] = details.get('run_time')
         self.context['document'] = details.get('document_list')
         self.context['image'] = details.get('image_list')
+        self.context['audio'] = details.get('audio_list')
         self.status = details.get('status')
         self.err_message = details.get('err_message')
         for key, value in workflow_variable.items():
@@ -57,7 +58,8 @@ class BaseStartStepNode(IStarNode):
         node_variable = {
             'question': question,
             'image': self.workflow_manage.image_list,
-            'document': self.workflow_manage.document_list
+            'document': self.workflow_manage.document_list,
+            'audio': self.workflow_manage.audio_list
         }
         return NodeResult(node_variable, workflow_variable)
 
@@ -80,5 +82,6 @@ class BaseStartStepNode(IStarNode):
             'err_message': self.err_message,
             'image_list': self.context.get('image'),
             'document_list': self.context.get('document'),
+            'audio_list': self.context.get('audio'),
             'global_fields': global_fields
         }

apps/application/flow/step_node/text_to_speech_step_node/__init__.py

@@ -0,0 +1,3 @@
+# coding=utf-8
+
+from .impl import *

apps/application/flow/step_node/text_to_speech_step_node/i_text_to_speech_node.py

@@ -0,0 +1,35 @@
+# coding=utf-8
+
+from typing import Type
+
+from rest_framework import serializers
+
+from application.flow.i_step_node import INode, NodeResult
+from common.util.field_message import ErrMessage
+
+
+class TextToSpeechNodeSerializer(serializers.Serializer):
+    tts_model_id = serializers.CharField(required=True, error_messages=ErrMessage.char("模型id"))
+
+    is_result = serializers.BooleanField(required=False, error_messages=ErrMessage.boolean('是否返回内容'))
+
+    content_list = serializers.ListField(required=False, error_messages=ErrMessage.list("文本内容"))
+    model_params_setting = serializers.DictField(required=False,
+                                                 error_messages=ErrMessage.integer("模型参数相关设置"))
+
+
+class ITextToSpeechNode(INode):
+    type = 'text-to-speech-node'
+
+    def get_node_params_serializer_class(self) -> Type[serializers.Serializer]:
+        return TextToSpeechNodeSerializer
+
+    def _run(self):
+        content = self.workflow_manage.get_reference_field(self.node_params_serializer.data.get('content_list')[0],
+                                                           self.node_params_serializer.data.get('content_list')[1:])
+        return self.execute(content=content, **self.node_params_serializer.data, **self.flow_params_serializer.data)
+
+    def execute(self, tts_model_id, chat_id,
+                content, model_params_setting=None,
+                **kwargs) -> NodeResult:
+        pass

apps/application/flow/step_node/text_to_speech_step_node/impl/__init__.py

@@ -0,0 +1,3 @@
+# coding=utf-8
+
+from .base_text_to_speech_node import BaseTextToSpeechNode

apps/application/flow/step_node/text_to_speech_step_node/impl/base_text_to_speech_node.py

@@ -0,0 +1,73 @@
+# coding=utf-8
+import io
+import mimetypes
+
+from django.core.files.uploadedfile import InMemoryUploadedFile
+
+from application.flow.i_step_node import NodeResult, INode
+from application.flow.step_node.image_understand_step_node.i_image_understand_node import IImageUnderstandNode
+from application.flow.step_node.text_to_speech_step_node.i_text_to_speech_node import ITextToSpeechNode
+from dataset.models import File
+from dataset.serializers.file_serializers import FileSerializer
+from setting.models_provider.tools import get_model_instance_by_model_user_id
+
+
+def bytes_to_uploaded_file(file_bytes, file_name="generated_audio.mp3"):
+    content_type, _ = mimetypes.guess_type(file_name)
+    if content_type is None:
+        # 如果未能识别，设置为默认的二进制文件类型
+        content_type = "application/octet-stream"
+    # 创建一个内存中的字节流对象
+    file_stream = io.BytesIO(file_bytes)
+
+    # 获取文件大小
+    file_size = len(file_bytes)
+
+    uploaded_file = InMemoryUploadedFile(
+        file=file_stream,
+        field_name=None,
+        name=file_name,
+        content_type=content_type,
+        size=file_size,
+        charset=None,
+    )
+    return uploaded_file
+
+
+class BaseTextToSpeechNode(ITextToSpeechNode):
+    def save_context(self, details, workflow_manage):
+        self.context['answer'] = details.get('answer')
+        self.answer_text = details.get('answer')
+
+    def execute(self, tts_model_id, chat_id,
+                content, model_params_setting=None,
+                **kwargs) -> NodeResult:
+        self.context['content'] = content
+        model = get_model_instance_by_model_user_id(tts_model_id, self.flow_params_serializer.data.get('user_id'),
+                                                    **model_params_setting)
+        audio_byte = model.text_to_speech(content)
+        # 需要把这个音频文件存储到数据库中
+        file_name = 'generated_audio.mp3'
+        file = bytes_to_uploaded_file(audio_byte, file_name)
+        application = self.workflow_manage.work_flow_post_handler.chat_info.application
+        meta = {
+            'debug': False if application.id else True,
+            'chat_id': chat_id,
+            'application_id': str(application.id) if application.id else None,
+        }
+        file_url = FileSerializer(data={'file': file, 'meta': meta}).upload()
+        # 拼接一个audio标签的src属性
+        audio_label = f'<audio src="{file_url}" controls style = "width: 300px; height: 43px" class ="border-r-4"/>'
+        return NodeResult({'answer': audio_label, 'result': audio_label}, {})
+
+    def get_details(self, index: int, **kwargs):
+        return {
+            'name': self.node.properties.get('stepName'),
+            "index": index,
+            'run_time': self.context.get('run_time'),
+            'type': self.node.type,
+            'status': self.status,
+            'content': self.context.get('content'),
+            'err_message': self.err_message,
+            'answer': self.context.get('answer'),
+        }

apps/application/flow/workflow_manage.py

@@ -54,7 +54,7 @@ class Node:
 
 
 end_nodes = ['ai-chat-node', 'reply-node', 'function-node', 'function-lib-node', 'application-node',
-             'image-understand-node', 'image-generate-node']
+             'image-understand-node', 'speech-to-text-node', 'text-to-speech-node', 'image-generate-node']
 
 
 class Flow:
@@ -244,6 +244,7 @@ class WorkflowManage:
     def __init__(self, flow: Flow, params, work_flow_post_handler: WorkFlowPostHandler,
                  base_to_response: BaseToResponse = SystemToResponse(), form_data=None, image_list=None,
                  document_list=None,
+                 audio_list=None,
                  start_node_id=None,
                  start_node_data=None, chat_record=None, child_node=None):
         if form_data is None:
@@ -252,11 +253,14 @@ class WorkflowManage:
             image_list = []
         if document_list is None:
             document_list = []
+        if audio_list is None:
+            audio_list = []
         self.start_node_id = start_node_id
         self.start_node = None
         self.form_data = form_data
         self.image_list = image_list
         self.document_list = document_list
+        self.audio_list = audio_list
         self.params = params
         self.flow = flow
         self.lock = threading.Lock()

apps/application/serializers/chat_message_serializers.py

@@ -245,6 +245,7 @@ class ChatMessageSerializer(serializers.Serializer):
     form_data = serializers.DictField(required=False, error_messages=ErrMessage.char("全局变量"))
     image_list = serializers.ListField(required=False, error_messages=ErrMessage.list("图片"))
     document_list = serializers.ListField(required=False, error_messages=ErrMessage.list("文档"))
+    audio_list = serializers.ListField(required=False, error_messages=ErrMessage.list("音频"))
     child_node = serializers.DictField(required=False, allow_null=True, error_messages=ErrMessage.dict("子节点"))
 
     def is_valid_application_workflow(self, *, raise_exception=False):
@@ -338,6 +339,7 @@ class ChatMessageSerializer(serializers.Serializer):
         form_data = self.data.get('form_data')
         image_list = self.data.get('image_list')
         document_list = self.data.get('document_list')
+        audio_list = self.data.get('audio_list')
         user_id = chat_info.application.user_id
         chat_record_id = self.data.get('chat_record_id')
         chat_record = None
@@ -354,7 +356,7 @@ class ChatMessageSerializer(serializers.Serializer):
                                            'client_id': client_id,
                                            'client_type': client_type,
                                            'user_id': user_id}, WorkFlowPostHandler(chat_info, client_id, client_type),
-                                          base_to_response, form_data, image_list, document_list,
+                                          base_to_response, form_data, image_list, document_list, audio_list,
                                           self.data.get('runtime_node_id'),
                                           self.data.get('node_data'), chat_record, self.data.get('child_node'))
         r = work_flow_manage.run()

apps/application/views/chat_views.py

@@ -134,6 +134,8 @@ class ChatView(APIView):
                                                    'image_list') if 'image_list' in request.data else [],
                                                'document_list': request.data.get(
                                                    'document_list') if 'document_list' in request.data else [],
+                                               'audio_list': request.data.get(
+                                                   'audio_list') if 'audio_list' in request.data else [],
                                                'client_type': request.auth.client_type,
                                                'node_id': request.data.get('node_id', None),
                                                'runtime_node_id': request.data.get('runtime_node_id', None),

apps/common/util/common.py

@@ -8,13 +8,15 @@
 """
 import hashlib
 import importlib
-import mimetypes
 import io
+import shutil
+import mimetypes
 from functools import reduce
 from typing import Dict, List
 
 from django.core.files.uploadedfile import InMemoryUploadedFile
 from django.db.models import QuerySet
+from pydub import AudioSegment
 
 from ..exception.app_exception import AppApiException
 from ..models.db_model_manage import DBModelManage
@@ -136,3 +138,61 @@ def bytes_to_uploaded_file(file_bytes, file_name="file.txt"):
         charset=None,
     )
     return uploaded_file
+
+def any_to_amr(any_path, amr_path):
+    """
+    把任意格式转成amr文件
+    """
+    if any_path.endswith(".amr"):
+        shutil.copy2(any_path, amr_path)
+        return
+    if any_path.endswith(".sil") or any_path.endswith(".silk") or any_path.endswith(".slk"):
+        raise NotImplementedError("Not support file type: {}".format(any_path))
+    audio = AudioSegment.from_file(any_path)
+    audio = audio.set_frame_rate(8000)  # only support 8000
+    audio.export(amr_path, format="amr")
+    return audio.duration_seconds * 1000
+
+
+def any_to_mp3(any_path, mp3_path):
+    """
+    把任意格式转成mp3文件
+    """
+    if any_path.endswith(".mp3"):
+        shutil.copy2(any_path, mp3_path)
+        return
+    if any_path.endswith(".sil") or any_path.endswith(".silk") or any_path.endswith(".slk"):
+        sil_to_wav(any_path, any_path)
+        any_path = mp3_path
+    audio = AudioSegment.from_file(any_path)
+    audio.export(mp3_path, format="mp3")
+
+
+def sil_to_wav(silk_path, wav_path, rate: int = 24000):
+    """
+    silk 文件转 wav
+    """
+    try:
+        import pysilk
+    except ImportError:
+        raise AppApiException("import pysilk failed, wechaty voice message will not be supported.")
+    wav_data = pysilk.decode_file(silk_path, to_wav=True, sample_rate=rate)
+    with open(wav_path, "wb") as f:
+        f.write(wav_data)
+
+
+def split_and_transcribe(file_path, model, max_segment_length_ms=59000, format="mp3"):
+    audio_data = AudioSegment.from_file(file_path, format=format)
+    audio_length_ms = len(audio_data)
+
+    if audio_length_ms <= max_segment_length_ms:
+        return model.speech_to_text(io.BytesIO(audio_data.export(format=format).read()))
+
+    full_text = []
+    for start_ms in range(0, audio_length_ms, max_segment_length_ms):
+        end_ms = min(audio_length_ms, start_ms + max_segment_length_ms)
+        segment = audio_data[start_ms:end_ms]
+        text = model.speech_to_text(io.BytesIO(segment.export(format=format).read()))
+        if isinstance(text, str):
+            full_text.append(text)
+    return ' '.join(full_text)

apps/dataset/serializers/file_serializers.py

@@ -77,5 +77,9 @@ class FileSerializer(serializers.Serializer):
             file = QuerySet(File).filter(id=file_id).first()
             if file is None:
                 raise NotFound404(404, "不存在的文件")
+            # 如果是mp3文件，直接返回文件流
+            if file.file_name.split(".")[-1] == 'mp3':
+                return HttpResponse(file.get_byte(), status=200, headers={'Content-Type': 'audio/mp3',
+                                                                          'Content-Disposition': 'attachment; filename="abc.mp3"'})
             return HttpResponse(file.get_byte(), status=200,
                                 headers={'Content-Type': mime_types.get(file.file_name.split(".")[-1], 'text/plain')})

apps/dataset/views/file.py

@@ -36,8 +36,8 @@ class FileView(APIView):
 
     class Operate(APIView):
         @action(methods=['GET'], detail=False)
-        @swagger_auto_schema(operation_summary="获取图片",
-                             operation_id="获取图片",
+        @swagger_auto_schema(operation_summary="获取文件",
+                             operation_id="获取文件",
                              tags=["文件"])
         def get(self, request: Request, file_id: str):
             return FileSerializer.Operate(data={'id': file_id}).get()

pyproject.toml

@@ -54,6 +54,10 @@ django-celery-beat = "^2.6.0"
 celery-once = "^3.0.1"
 anthropic = "^0.34.2"
 pylint = "3.1.0"
+ffmpeg-python = "^0.2.0"
+pydub = "^0.25.1"
+cffi = "^1.17.1"
+pysilk = "^0.0.1"
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

ui/src/api/type/application.ts

@@ -63,6 +63,7 @@ interface chatType {
   upload_meta?: {
     document_list: Array<any>
     image_list: Array<any>
+    audio_list: Array<any>
   }
 }
 

ui/src/assets/icon_speech_to_text.svg

@@ -0,0 +1,11 @@
+<svg width="20" height="20" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0 8C0 3.58172 3.58172 0 8 0H24C28.4183 0 32 3.58172 32 8V24C32 28.4183 28.4183 32 24 32H8C3.58172 32 0 28.4183 0 24V8Z" fill="#FF8800"/>
+<g clip-path="url(#clip0_10330_2372)">
+<path d="M22.5715 7.55565C22.7761 7.55565 22.9542 7.69533 23.0031 7.89401L26.3667 21.5792C26.4356 21.8593 26.2235 22.1297 25.9351 22.1297H24.9468C24.7483 22.1297 24.5739 21.9981 24.5194 21.8072L23.4149 17.9355H19.6078L19.0541 19.8045C18.1413 22.4957 15.4533 24.4445 12.2798 24.4445C9.32693 24.4445 6.79442 22.7572 5.71945 20.3549C5.69242 20.2945 5.66068 20.2159 5.62896 20.134C5.53599 19.8941 5.67535 19.6296 5.92345 19.5612C5.96516 19.5497 6.0032 19.5393 6.03397 19.5308C6.35482 19.4424 6.61238 19.3715 6.80666 19.318C7.02895 19.2568 7.25552 19.3785 7.34889 19.5893C8.14894 21.3957 10.0553 22.6668 12.2798 22.6668C14.5329 22.6668 16.4597 21.3628 17.2411 19.5193L17.317 19.3285L20.0773 7.89578C20.1255 7.69624 20.3041 7.55565 20.5094 7.55565H22.5715ZM12.2798 7.11121C14.9799 7.11121 17.1687 9.41058 17.1687 12.247V16.1976C17.1687 19.0341 14.9799 21.3334 12.2798 21.3334C9.57975 21.3334 7.39092 19.0341 7.39092 16.1976V12.247C7.39092 9.41058 9.57975 7.11121 12.2798 7.11121ZM21.5396 9.10632L19.9645 16.0903H23.1147L21.5396 9.10632Z" fill="white"/>
+</g>
+<defs>
+<clipPath id="clip0_10330_2372">
+<rect width="21.3333" height="21.3333" fill="white" transform="translate(5.33331 5.33337)"/>
+</clipPath>
+</defs>
+</svg>

ui/src/assets/icon_text_to_speech.svg

@@ -0,0 +1,11 @@
+<svg width="20" height="20" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0 8C0 3.58172 3.58172 0 8 0H24C28.4183 0 32 3.58172 32 8V24C32 28.4183 28.4183 32 24 32H8C3.58172 32 0 28.4183 0 24V8Z" fill="#14C0FF"/>
+<g clip-path="url(#clip0_10330_2180)">
+<path d="M9.42854 7.55565C9.22395 7.55565 9.04578 7.69533 8.99695 7.89401L5.63329 21.5792C5.56444 21.8593 5.77647 22.1297 6.06488 22.1297H7.05321C7.25171 22.1297 7.42615 21.9981 7.4806 21.8072L8.5851 17.9355H12.3922L12.9459 19.8045C13.8587 22.4957 16.5467 24.4445 19.7202 24.4445C22.6731 24.4445 25.2056 22.7572 26.2805 20.3549C26.3076 20.2945 26.3393 20.2159 26.371 20.134C26.464 19.8941 26.3246 19.6296 26.0765 19.5612C26.0348 19.5497 25.9968 19.5393 25.966 19.5308C25.6452 19.4424 25.3876 19.3715 25.1933 19.318C24.9711 19.2568 24.7445 19.3785 24.6511 19.5893C23.8511 21.3957 21.9447 22.6668 19.7202 22.6668C17.4671 22.6668 15.5403 21.3628 14.7589 19.5193L14.683 19.3285L11.9227 7.89578C11.8745 7.69624 11.6959 7.55565 11.4906 7.55565H9.42854ZM19.7202 7.11121C17.0201 7.11121 14.8313 9.41058 14.8313 12.247V16.1976C14.8313 19.0341 17.0201 21.3334 19.7202 21.3334C22.4203 21.3334 24.6091 19.0341 24.6091 16.1976V12.247C24.6091 9.41058 22.4203 7.11121 19.7202 7.11121ZM10.4604 9.10632L12.0355 16.0903H8.88529L10.4604 9.10632Z" fill="white"/>
+</g>
+<defs>
+<clipPath id="clip0_10330_2180">
+<rect width="21.3333" height="21.3333" fill="white" transform="matrix(-1 0 0 1 26.6667 5.33337)"/>
+</clipPath>
+</defs>
+</svg>

ui/src/components/ai-chat/ExecutionDetailDialog.vue

@@ -14,9 +14,9 @@
           <el-card class="mb-8" shadow="never" style="--el-card-padding: 12px 16px">
             <div class="flex-between cursor" @click="current = current === index ? '' : index">
               <div class="flex align-center">
-                <el-icon class="mr-8 arrow-icon" :class="current === index ? 'rotate-90' : ''"
-                  ><CaretRight
-                /></el-icon>
+                <el-icon class="mr-8 arrow-icon" :class="current === index ? 'rotate-90' : ''">
+                  <CaretRight />
+                </el-icon>
                 <component
                   :is="iconComponent(`${item.type}-icon`)"
                   class="mr-8"
@@ -38,10 +38,12 @@
                   >{{ item?.message_tokens + item?.answer_tokens }} tokens</span
                 >
                 <span class="mr-16 color-secondary">{{ item?.run_time?.toFixed(2) || 0.0 }} s</span>
-                <el-icon class="success" :size="16" v-if="item.status === 200"
-                  ><CircleCheck
-                /></el-icon>
-                <el-icon class="danger" :size="16" v-else><CircleClose /></el-icon>
+                <el-icon class="success" :size="16" v-if="item.status === 200">
+                  <CircleCheck />
+                </el-icon>
+                <el-icon class="danger" :size="16" v-else>
+                  <CircleClose />
+                </el-icon>
               </div>
             </div>
             <el-collapse-transition>
@@ -98,6 +100,20 @@
                             </template>
                           </el-space>
                         </div>
+                        <div v-if="item.audio_list?.length > 0">
+                          <p class="mb-8 color-secondary">语音文件:</p>
+
+                          <el-space wrap>
+                            <template v-for="(f, i) in item.audio_list" :key="i">
+                              <audio
+                                :src="f.url"
+                                controls
+                                style="width: 300px; height: 43px"
+                                class="border-r-4"
+                              />
+                            </template>
+                          </el-space>
+                        </div>
                       </div>
                     </div>
                   </template>
@@ -122,7 +138,7 @@
                             <ParagraphCard :data="paragraph" :index="paragraphIndex" />
                           </template>
                         </template>
-                        <template v-else> - </template>
+                        <template v-else> -</template>
                       </div>
                     </div>
                   </template>
@@ -168,7 +184,7 @@
                             ><span>{{ history.content }}</span>
                           </p>
                         </template>
-                        <template v-else> - </template>
+                        <template v-else> -</template>
                       </div>
                     </div>
                     <div
@@ -192,7 +208,7 @@
                           :modelValue="item.answer"
                           style="background: none"
                         />
-                        <template v-else> - </template>
+                        <template v-else> -</template>
                       </div>
                     </div>
                   </template>
@@ -210,7 +226,7 @@
                             :modelValue="item.answer"
                             style="background: none"
                           />
-                          <template v-else> - </template>
+                          <template v-else> -</template>
                         </el-scrollbar>
                       </div>
                     </div>
@@ -246,12 +262,75 @@
                               :modelValue="file_content"
                               style="background: none"
                             />
-                            <template v-else> - </template>
+                            <template v-else> -</template>
                           </el-card>
                         </el-scrollbar>
                       </div>
                     </div>
                   </template>
+                  <template v-if="item.type === WorkflowType.SpeechToTextNode">
+                    <div class="card-never border-r-4">
+                      <h5 class="p-8-12">参数输入</h5>
+                      <div class="p-8-12 border-t-dashed lighter">
+                        <div class="mb-8">
+                          <div v-if="item.audio_list?.length > 0">
+                            <p class="mb-8 color-secondary">语音文件:</p>
+
+                            <el-space wrap>
+                              <template v-for="(f, i) in item.audio_list" :key="i">
+                                <audio
+                                  :src="f.url"
+                                  controls
+                                  style="width: 300px; height: 43px"
+                                  class="border-r-4"
+                                />
+                              </template>
+                            </el-space>
+                          </div>
+                        </div>
+                      </div>
+                    </div>
+                    <div class="card-never border-r-4">
+                      <h5 class="p-8-12">参数输出</h5>
+                      <div class="p-8-12 border-t-dashed lighter">
+                        <p class="mb-8 color-secondary">文本内容:</p>
+                        <div v-if="item.answer">
+                          <MdPreview
+                            ref="editorRef"
+                            editorId="preview-only"
+                            :modelValue="item.answer"
+                            style="background: none"
+                          />
+                        </div>
+                      </div>
+                    </div>
+                  </template>
+
+                  <template v-if="item.type === WorkflowType.TextToSpeechNode">
+                    <div class="card-never border-r-4">
+                      <h5 class="p-8-12">参数输入</h5>
+                      <div class="p-8-12 border-t-dashed lighter">
+                        <div class="p-8-12 border-t-dashed lighter">
+                          <p class="mb-8 color-secondary">文本内容:</p>
+                          <div v-if="item.content">
+                            <MdPreview
+                              ref="editorRef"
+                              editorId="preview-only"
+                              :modelValue="item.content"
+                              style="background: none"
+                            />
+                          </div>
+                        </div>
+                      </div>
+                    </div>
+                    <div class="card-never border-r-4">
+                      <h5 class="p-8-12">参数输出</h5>
+                      <div class="p-8-12 border-t-dashed lighter">
+                        <p class="mb-8 color-secondary">语音文件:</p>
+                        <div v-if="item.answer" v-html="item.answer"></div>
+                      </div>
+                    </div>
+                  </template>
 
                   <!-- 函数库 -->
                   <template
@@ -300,7 +379,7 @@
                             </CardBox>
                           </template>
                         </template>
-                        <template v-else> - </template>
+                        <template v-else> -</template>
                       </div>
                     </div>
                     <div class="card-never border-r-4 mt-8">
@@ -334,7 +413,7 @@
                             </CardBox>
                           </template>
                         </template>
-                        <template v-else> - </template>
+                        <template v-else> -</template>
                       </div>
                     </div>
                   </template>
@@ -405,7 +484,7 @@
                             <span v-else>{{ history.content }}</span>
                           </p>
                         </template>
-                        <template v-else> - </template>
+                        <template v-else> -</template>
                       </div>
                     </div>
                     <div class="card-never border-r-4 mt-8">
@@ -441,7 +520,7 @@
                           :modelValue="item.answer"
                           style="background: none"
                         />
-                        <template v-else> - </template>
+                        <template v-else> -</template>
                       </div>
                     </div>
                   </template>
@@ -528,6 +607,7 @@ import { iconComponent } from '@/workflow/icons/utils'
 import { WorkflowType } from '@/enums/workflow'
 import { getImgUrl } from '@/utils/utils'
 import DynamicsForm from '@/components/dynamics-form/index.vue'
+
 const dialogVisible = ref(false)
 const detail = ref<any[]>([])
 
@@ -555,19 +635,24 @@ defineExpose({ open })
   .el-dialog__header {
     padding: 24px 24px 0 24px;
   }
+
   .el-dialog__body {
     padding: 8px !important;
   }
+
   .execution-details {
     max-height: calc(100vh - 260px);
+
     .arrow-icon {
       transition: 0.2s;
     }
   }
 }
+
 @media only screen and (max-width: 768px) {
   .execution-details-dialog {
     width: 90% !important;
+
     .footer-content {
       display: block;
     }

ui/src/components/ai-chat/component/question-content/index.vue

@@ -55,6 +55,20 @@
             </template>
           </el-space>
         </div>
+        <div class="mb-8" v-if="audio_list.length">
+          <el-space wrap>
+            <template v-for="(item, index) in audio_list" :key="index">
+              <div class="file cursor border-r-4" v-if="item.url">
+                <audio
+                  :src="item.url"
+                  controls
+                  style="width: 350px; height: 43px"
+                  class="border-r-4"
+                />
+              </div>
+            </template>
+          </el-space>
+        </div>
         {{ chatRecord.problem_text }}
       </div>
     </div>
@@ -87,6 +101,15 @@ const image_list = computed(() => {
   )
   return startNode?.image_list || []
 })
+const audio_list = computed(() => {
+  if (props.chatRecord?.upload_meta) {
+    return props.chatRecord.upload_meta?.audio_list || []
+  }
+  const startNode = props.chatRecord.execution_details?.find(
+    (detail) => detail.type === 'start-node'
+  )
+  return startNode?.audio_list || []
+})
 
 function downloadFile(item: any) {
   downloadByURL(item.url, item.name)

ui/src/components/ai-chat/index.vue

@@ -293,7 +293,9 @@ function chatMessage(chat?: any, problem?: string, re_chat?: boolean, other_para
         document_list:
           other_params_data && other_params_data.document_list
             ? other_params_data.document_list
-            : []
+            : [],
+        audio_list:
+          other_params_data && other_params_data.audio_list ? other_params_data.audio_list : []
       }
     })
     chatList.value.push(chat)

ui/src/enums/workflow.ts

@@ -12,6 +12,8 @@ export enum WorkflowType {
   Application = 'application-node',
   DocumentExtractNode = 'document-extract-node',
   ImageUnderstandNode = 'image-understand-node',
-  ImageGenerateNode = 'image-generate-node',
-  FormNode = 'form-node'
+  FormNode = 'form-node',
+  TextToSpeechNode = 'text-to-speech-node',
+  SpeechToTextNode = 'speech-to-text-node',
+    ImageGenerateNode = 'image-generate-node'
 }

ui/src/views/application-workflow/component/DropdownMenu.vue

@@ -171,7 +171,8 @@ function clickNodes(item: any, data?: any, type?: string) {
             ? {}
             : {
                 ...(fileUploadSetting.document ? { document_list: [] } : {}),
-                ...(fileUploadSetting.image ? { image_list: [] } : {})
+                ...(fileUploadSetting.image ? { image_list: [] } : {}),
+                ...(fileUploadSetting.audio ? { audio_list: [] } : {})
               })
         }
       } else {
@@ -215,7 +216,8 @@ function onmousedown(item: any, data?: any, type?: string) {
             ? {}
             : {
                 ...(fileUploadSetting.document ? { document_list: [] } : {}),
-                ...(fileUploadSetting.image ? { image_list: [] } : {})
+                ...(fileUploadSetting.image ? { image_list: [] } : {}),
+                ...(fileUploadSetting.audio ? { audio_list: [] } : {})
               })
         }
       } else {

ui/src/workflow/common/data.ts

@@ -250,6 +250,41 @@ export const imageGenerateNode = {
     }
   }
 }
+
+export const speechToTextNode = {
+  type: WorkflowType.SpeechToTextNode,
+  text: '将音频通过语音识别模型转换为文本',
+  label: '语音转文本',
+  height: 252,
+  properties: {
+    stepName: '语音转文本',
+    config: {
+      fields: [
+        {
+          label: '结果',
+          value: 'result'
+        }
+      ]
+    }
+  }
+}
+export const textToSpeechNode = {
+  type: WorkflowType.TextToSpeechNode,
+  text: '将文本通过语音合成模型转换为音频文件',
+  label: '文本转语音',
+  height: 252,
+  properties: {
+    stepName: '文本转语音',
+    config: {
+      fields: [
+        {
+          label: '结果',
+          value: 'result'
+        }
+      ]
+    }
+  }
+}
 export const menuNodes = [
   aiChatNode,
   searchDatasetNode,
@@ -259,8 +294,10 @@ export const menuNodes = [
   rerankerNode,
   documentExtractNode,
   imageUnderstandNode,
-  imageGenerateNode,
-  formNode
+  formNode,
+  speechToTextNode,
+  textToSpeechNode,
+      imageGenerateNode
 ]
 
 /**
@@ -351,7 +388,9 @@ export const nodeDict: any = {
   [WorkflowType.Application]: applicationNode,
   [WorkflowType.DocumentExtractNode]: documentExtractNode,
   [WorkflowType.ImageUnderstandNode]: imageUnderstandNode,
-  [WorkflowType.ImageGenerateNode]: imageGenerateNode
+  [WorkflowType.TextToSpeechNode]: textToSpeechNode,
+  [WorkflowType.SpeechToTextNode]: speechToTextNode,
+    [WorkflowType.ImageGenerateNode]: imageGenerateNode
 }
 export function isWorkFlow(type: string | undefined) {
   return type === 'WORK_FLOW'

ui/src/workflow/common/validate.ts

@@ -6,8 +6,10 @@ const end_nodes: Array<string> = [
   WorkflowType.FunctionLib,
   WorkflowType.FunctionLibCustom,
   WorkflowType.ImageUnderstandNode,
-  WorkflowType.ImageGenerateNode,
-  WorkflowType.Application
+  WorkflowType.Application,
+  WorkflowType.SpeechToTextNode,
+  WorkflowType.TextToSpeechNode,
+    WorkflowType.ImageGenerateNode,
 ]
 export class WorkFlowInstance {
   nodes

ui/src/workflow/icons/speech-to-text-node-icon.vue

@@ -0,0 +1,6 @@
+<template>
+  <AppAvatar shape="square" style="background: #ff8800">
+    <img src="@/assets/icon_speech_to_text.svg" style="width: 100%" alt="" />
+  </AppAvatar>
+</template>
+<script setup lang="ts"></script>

ui/src/workflow/icons/text-to-speech-node-icon.vue

@@ -0,0 +1,6 @@
+<template>
+  <AppAvatar shape="square" style="background: #14c0ff">
+    <img src="@/assets/icon_text_to_speech.svg" style="width: 100%" alt="" />
+  </AppAvatar>
+</template>
+<script setup lang="ts"></script>

ui/src/workflow/nodes/application-node/index.vue

@@ -33,7 +33,7 @@
           label="选择文档"
           prop="document_list"
           :rules="{
-            message: '请选择检索问题',
+            message: '请选择文档',
             trigger: 'blur',
             required: false
           }"
@@ -42,7 +42,7 @@
             ref="nodeCascaderRef"
             :nodeModel="nodeModel"
             class="w-full"
-            placeholder="请选择检索问题"
+            placeholder="请选择文档"
             v-model="form_data.document_list"
           />
         </el-form-item>
@@ -52,7 +52,7 @@
           label="选择图片"
           prop="image_list"
           :rules="{
-            message: '请选择检索问题',
+            message: '请选择图片',
             trigger: 'blur',
             required: false
           }"
@@ -61,10 +61,29 @@
             ref="nodeCascaderRef"
             :nodeModel="nodeModel"
             class="w-full"
-            placeholder="请选择检索问题"
+            placeholder="请选择图片"
             v-model="form_data.image_list"
           />
         </el-form-item>
+
+        <el-form-item
+          v-if="form_data.hasOwnProperty('audio_list') || 'audio_list' in form_data"
+          label="选择语音文件"
+          prop="audio_list"
+          :rules="{
+            message: '请选择语音文件',
+            trigger: 'blur',
+            required: false
+          }"
+        >
+          <NodeCascader
+            ref="nodeCascaderRef"
+            :nodeModel="nodeModel"
+            class="w-full"
+            placeholder="请选择语音文件"
+            v-model="form_data.audio_list"
+          />
+        </el-form-item>
         <div v-for="(field, index) in form_data.api_input_field_list" :key="'api-input-' + index">
           <el-form-item
             :label="field.variable"
@@ -135,7 +154,8 @@ const form = {
   api_input_field_list: [],
   user_input_field_list: [],
   document_list: ['start-node', 'document'],
-  image_list: ['start-node', 'image']
+  image_list: ['start-node', 'image'],
+  audio_list: ['start-node', 'audio']
 }
 
 const applicationNodeFormRef = ref<FormInstance>()

ui/src/workflow/nodes/speech-to-text-node/index.ts

@@ -0,0 +1,12 @@
+import SpeechToTextVue from './index.vue'
+import { AppNode, AppNodeModel } from '@/workflow/common/app-node'
+class SpeechToTextNode extends AppNode {
+  constructor(props: any) {
+    super(props, SpeechToTextVue)
+  }
+}
+export default {
+  type: 'speech-to-text-node',
+  model: AppNodeModel,
+  view: SpeechToTextNode
+}

ui/src/workflow/nodes/speech-to-text-node/index.vue

@@ -0,0 +1,225 @@
+<template>
+  <NodeContainer :node-model="nodeModel">
+    <h5 class="title-decoration-1 mb-8">节点设置</h5>
+    <el-card shadow="never" class="card-never">
+      <el-form
+        @submit.prevent
+        :model="form_data"
+        label-position="top"
+        require-asterisk-position="right"
+        label-width="auto"
+        ref="aiChatNodeFormRef"
+        hide-required-asterisk
+      >
+        <el-form-item
+          label="语音识别模型"
+          prop="stt_model_id"
+          :rules="{
+            required: true,
+            message: '请选择语音识别模型',
+            trigger: 'change'
+          }"
+        >
+          <template #label>
+            <div class="flex-between w-full">
+              <div>
+                <span>语音识别模型<span class="danger">*</span></span>
+              </div>
+            </div>
+          </template>
+          <el-select
+            @change="model_change"
+            @wheel="wheel"
+            :teleported="false"
+            v-model="form_data.stt_model_id"
+            placeholder="请选择语音识别模型"
+            class="w-full"
+            popper-class="select-model"
+            :clearable="true"
+          >
+            <el-option-group
+              v-for="(value, label) in modelOptions"
+              :key="value"
+              :label="relatedObject(providerOptions, label, 'provider')?.name"
+            >
+              <el-option
+                v-for="item in value.filter((v: any) => v.status === 'SUCCESS')"
+                :key="item.id"
+                :label="item.name"
+                :value="item.id"
+                class="flex-between"
+              >
+                <div class="flex align-center">
+                  <span
+                    v-html="relatedObject(providerOptions, label, 'provider')?.icon"
+                    class="model-icon mr-8"
+                  ></span>
+                  <span>{{ item.name }}</span>
+                  <el-tag v-if="item.permission_type === 'PUBLIC'" type="info" class="info-tag ml-8"
+                    >公用
+                  </el-tag>
+                </div>
+                <el-icon class="check-icon" v-if="item.id === form_data.stt_model_id">
+                  <Check />
+                </el-icon>
+              </el-option>
+              <!-- 不可用 -->
+              <el-option
+                v-for="item in value.filter((v: any) => v.status !== 'SUCCESS')"
+                :key="item.id"
+                :label="item.name"
+                :value="item.id"
+                class="flex-between"
+                disabled
+              >
+                <div class="flex">
+                  <span
+                    v-html="relatedObject(providerOptions, label, 'provider')?.icon"
+                    class="model-icon mr-8"
+                  ></span>
+                  <span>{{ item.name }}</span>
+                  <span class="danger">（不可用）</span>
+                </div>
+                <el-icon class="check-icon" v-if="item.id === form_data.stt_model_id">
+                  <Check />
+                </el-icon>
+              </el-option>
+            </el-option-group>
+          </el-select>
+        </el-form-item>
+        <el-form-item
+          label="选择语音文件"
+          prop="audio_list"
+          :rules="{
+            message: '选择语音文件',
+            trigger: 'blur',
+            required: true
+          }"
+        >
+          <template #label>
+            <div class="flex-between w-full">
+              <div>
+                <span>选择语音文件<span class="danger">*</span></span>
+              </div>
+            </div>
+          </template>
+          <NodeCascader
+            ref="nodeCascaderRef"
+            :nodeModel="nodeModel"
+            class="w-full"
+            placeholder="请选择语音文件"
+            v-model="form_data.audio_list"
+          />
+        </el-form-item>
+
+        <el-form-item label="返回内容" @click.prevent>
+          <template #label>
+            <div class="flex align-center">
+              <div class="mr-4">
+                <span>返回内容<span class="danger">*</span></span>
+              </div>
+              <el-tooltip effect="dark" placement="right" popper-class="max-w-200">
+                <template #content>
+                  关闭后该节点的内容则不输出给用户。
+                  如果你想让用户看到该节点的输出内容，请打开开关。
+                </template>
+                <AppIcon iconName="app-warning" class="app-warning-icon"></AppIcon>
+              </el-tooltip>
+            </div>
+          </template>
+          <el-switch size="small" v-model="form_data.is_result" />
+        </el-form-item>
+      </el-form>
+    </el-card>
+  </NodeContainer>
+</template>
+
+<script setup lang="ts">
+import NodeContainer from '@/workflow/common/NodeContainer.vue'
+import { computed, onMounted, ref } from 'vue'
+import { groupBy, set } from 'lodash'
+import { relatedObject } from '@/utils/utils'
+import type { Provider } from '@/api/type/model'
+import applicationApi from '@/api/application'
+import { app } from '@/main'
+import useStore from '@/stores'
+import NodeCascader from '@/workflow/common/NodeCascader.vue'
+import type { FormInstance } from 'element-plus'
+const { model } = useStore()
+
+const {
+  params: { id }
+} = app.config.globalProperties.$route as any
+
+const props = defineProps<{ nodeModel: any }>()
+const modelOptions = ref<any>(null)
+const providerOptions = ref<Array<Provider>>([])
+
+const aiChatNodeFormRef = ref<FormInstance>()
+const validate = () => {
+  return aiChatNodeFormRef.value?.validate().catch((err) => {
+    return Promise.reject({ node: props.nodeModel, errMessage: err })
+  })
+}
+
+const wheel = (e: any) => {
+  if (e.ctrlKey === true) {
+    e.preventDefault()
+    return true
+  } else {
+    e.stopPropagation()
+    return true
+  }
+}
+
+const form = {
+  stt_model_id: '',
+  is_result: true,
+  audio_list: ['start-node', 'audio']
+}
+
+const form_data = computed({
+  get: () => {
+    if (props.nodeModel.properties.node_data) {
+      return props.nodeModel.properties.node_data
+    } else {
+      set(props.nodeModel.properties, 'node_data', form)
+    }
+    return props.nodeModel.properties.node_data
+  },
+  set: (value) => {
+    set(props.nodeModel.properties, 'node_data', value)
+  }
+})
+
+function getModel() {
+  if (id) {
+    applicationApi.getApplicationSTTModel(id).then((res: any) => {
+      modelOptions.value = groupBy(res?.data, 'provider')
+    })
+  } else {
+    model.asyncGetModel().then((res: any) => {
+      modelOptions.value = groupBy(res?.data, 'provider')
+    })
+  }
+}
+
+function getProvider() {
+  model.asyncGetProvider().then((res: any) => {
+    providerOptions.value = res?.data
+  })
+}
+
+const model_change = (model_id?: string) => {
+  console.log(modelOptions.value)
+}
+
+onMounted(() => {
+  getModel()
+  getProvider()
+
+  set(props.nodeModel, 'validate', validate)
+})
+</script>
+
+<style scoped lang="scss"></style>

ui/src/workflow/nodes/text-to-speech-node/index.ts

@@ -0,0 +1,12 @@
+import TextToSpeechVue from './index.vue'
+import { AppNode, AppNodeModel } from '@/workflow/common/app-node'
+class TextToSpeechNode extends AppNode {
+  constructor(props: any) {
+    super(props, TextToSpeechVue)
+  }
+}
+export default {
+  type: 'text-to-speech-node',
+  model: AppNodeModel,
+  view: TextToSpeechNode
+}

ui/src/workflow/nodes/text-to-speech-node/index.vue

@@ -0,0 +1,250 @@
+<template>
+  <NodeContainer :node-model="nodeModel">
+    <h5 class="title-decoration-1 mb-8">节点设置</h5>
+    <el-card shadow="never" class="card-never">
+      <el-form
+        @submit.prevent
+        :model="form_data"
+        label-position="top"
+        require-asterisk-position="right"
+        label-width="auto"
+        ref="aiChatNodeFormRef"
+        hide-required-asterisk
+      >
+        <el-form-item
+          label="语音合成模型"
+          prop="tts_model_id"
+          :rules="{
+            required: true,
+            message: '请选择语音合成模型',
+            trigger: 'change'
+          }"
+        >
+          <template #label>
+            <div class="flex-between w-full">
+              <div>
+                <span>语音合成模型<span class="danger">*</span></span>
+              </div>
+              <el-button
+                type="primary"
+                link
+                @click="openTTSParamSettingDialog"
+                :disabled="!form_data.tts_model_id"
+                class="mr-4"
+              >
+                {{ $t('views.application.applicationForm.form.paramSetting') }}
+              </el-button>
+            </div>
+          </template>
+          <el-select
+            @change="model_change"
+            @wheel="wheel"
+            :teleported="false"
+            v-model="form_data.tts_model_id"
+            placeholder="请选择语音合成模型"
+            class="w-full"
+            popper-class="select-model"
+            :clearable="true"
+          >
+            <el-option-group
+              v-for="(value, label) in modelOptions"
+              :key="value"
+              :label="relatedObject(providerOptions, label, 'provider')?.name"
+            >
+              <el-option
+                v-for="item in value.filter((v: any) => v.status === 'SUCCESS')"
+                :key="item.id"
+                :label="item.name"
+                :value="item.id"
+                class="flex-between"
+              >
+                <div class="flex align-center">
+                  <span
+                    v-html="relatedObject(providerOptions, label, 'provider')?.icon"
+                    class="model-icon mr-8"
+                  ></span>
+                  <span>{{ item.name }}</span>
+                  <el-tag v-if="item.permission_type === 'PUBLIC'" type="info" class="info-tag ml-8"
+                    >公用
+                  </el-tag>
+                </div>
+                <el-icon class="check-icon" v-if="item.id === form_data.model_id">
+                  <Check />
+                </el-icon>
+              </el-option>
+              <!-- 不可用 -->
+              <el-option
+                v-for="item in value.filter((v: any) => v.status !== 'SUCCESS')"
+                :key="item.id"
+                :label="item.name"
+                :value="item.id"
+                class="flex-between"
+                disabled
+              >
+                <div class="flex">
+                  <span
+                    v-html="relatedObject(providerOptions, label, 'provider')?.icon"
+                    class="model-icon mr-8"
+                  ></span>
+                  <span>{{ item.name }}</span>
+                  <span class="danger">（不可用）</span>
+                </div>
+                <el-icon class="check-icon" v-if="item.id === form_data.model_id">
+                  <Check />
+                </el-icon>
+              </el-option>
+            </el-option-group>
+          </el-select>
+        </el-form-item>
+        <el-form-item
+          label="选择文本内容"
+          :rules="{
+            message: '选择文本内容',
+            trigger: 'blur',
+            required: true
+          }"
+        >
+          <template #label>
+            <div class="flex-between w-full">
+              <div>
+                <span>选择文本内容<span class="danger">*</span></span>
+              </div>
+            </div>
+          </template>
+          <NodeCascader
+            ref="nodeCascaderRef"
+            :nodeModel="nodeModel"
+            class="w-full"
+            placeholder="选择文本内容"
+            v-model="form_data.content_list"
+          />
+        </el-form-item>
+
+        <el-form-item label="返回内容" @click.prevent>
+          <template #label>
+            <div class="flex align-center">
+              <div class="mr-4">
+                <span>返回内容<span class="danger">*</span></span>
+              </div>
+              <el-tooltip effect="dark" placement="right" popper-class="max-w-200">
+                <template #content>
+                  关闭后该节点的内容则不输出给用户。
+                  如果你想让用户看到该节点的输出内容，请打开开关。
+                </template>
+                <AppIcon iconName="app-warning" class="app-warning-icon"></AppIcon>
+              </el-tooltip>
+            </div>
+          </template>
+          <el-switch size="small" v-model="form_data.is_result" />
+        </el-form-item>
+      </el-form>
+    </el-card>
+  </NodeContainer>
+  <TTSModeParamSettingDialog ref="TTSModeParamSettingDialogRef" @refresh="refreshTTSForm" />
+</template>
+
+<script setup lang="ts">
+import NodeContainer from '@/workflow/common/NodeContainer.vue'
+import { computed, onMounted, ref } from 'vue'
+import { groupBy, set } from 'lodash'
+import { relatedObject } from '@/utils/utils'
+import type { Provider } from '@/api/type/model'
+import applicationApi from '@/api/application'
+import { app } from '@/main'
+import useStore from '@/stores'
+import NodeCascader from '@/workflow/common/NodeCascader.vue'
+import type { FormInstance } from 'element-plus'
+import { MsgSuccess } from '@/utils/message'
+import { t } from '@/locales'
+import TTSModeParamSettingDialog from '@/views/application/component/TTSModeParamSettingDialog.vue'
+
+const TTSModeParamSettingDialogRef = ref<InstanceType<typeof TTSModeParamSettingDialog>>()
+const { model } = useStore()
+
+const {
+  params: { id }
+} = app.config.globalProperties.$route as any
+
+const props = defineProps<{ nodeModel: any }>()
+const modelOptions = ref<any>(null)
+const providerOptions = ref<Array<Provider>>([])
+
+const aiChatNodeFormRef = ref<FormInstance>()
+const validate = () => {
+  return aiChatNodeFormRef.value?.validate().catch((err) => {
+    return Promise.reject({ node: props.nodeModel, errMessage: err })
+  })
+}
+
+const wheel = (e: any) => {
+  if (e.ctrlKey === true) {
+    e.preventDefault()
+    return true
+  } else {
+    e.stopPropagation()
+    return true
+  }
+}
+
+const form = {
+  tts_model_id: '',
+  is_result: true,
+  content_list: ['start-node', 'content'],
+  model_params_setting: {}
+}
+
+const form_data = computed({
+  get: () => {
+    if (props.nodeModel.properties.node_data) {
+      return props.nodeModel.properties.node_data
+    } else {
+      set(props.nodeModel.properties, 'node_data', form)
+    }
+    return props.nodeModel.properties.node_data
+  },
+  set: (value) => {
+    set(props.nodeModel.properties, 'node_data', value)
+  }
+})
+
+function getModel() {
+  if (id) {
+    applicationApi.getApplicationTTSModel(id).then((res: any) => {
+      modelOptions.value = groupBy(res?.data, 'provider')
+    })
+  } else {
+    model.asyncGetModel().then((res: any) => {
+      modelOptions.value = groupBy(res?.data, 'provider')
+    })
+  }
+}
+
+function getProvider() {
+  model.asyncGetProvider().then((res: any) => {
+    providerOptions.value = res?.data
+  })
+}
+
+const openTTSParamSettingDialog = () => {
+  const model_id = form_data.value.tts_model_id
+  if (!model_id) {
+    MsgSuccess(t('请选择语音播放模型'))
+    return
+  }
+  TTSModeParamSettingDialogRef.value?.open(model_id, id, form_data.value.model_params_setting)
+}
+const refreshTTSForm = (data: any) => {
+  form_data.value.model_params_setting = data
+}
+
+const model_change = (model_id?: string) => {}
+
+onMounted(() => {
+  getModel()
+  getProvider()
+
+  set(props.nodeModel, 'validate', validate)
+})
+</script>
+
+<style scoped lang="scss"></style>