From 15ec70c4a8d6390e8dbe853c9e34afcd728a1ef3 Mon Sep 17 00:00:00 2001 From: zhangzhanwei Date: Fri, 15 Aug 2025 17:47:37 +0800 Subject: [PATCH] feat: Interface with Qwen Omni speech to text model --- apps/locales/en_US/LC_MESSAGES/django.po | 9 ++ apps/locales/zh_CN/LC_MESSAGES/django.po | 11 ++- apps/locales/zh_Hant/LC_MESSAGES/django.po | 11 ++- .../aliyun_bai_lian_model_provider.py | 8 +- .../credential/omi_stt.py | 72 +++++++++++++++ .../model/omi_stt.py | 87 +++++++++++++++++++ .../impl/tencent_model_provider/model/stt.py | 2 + 7 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/omi_stt.py create mode 100644 apps/models_provider/impl/aliyun_bai_lian_model_provider/model/omi_stt.py diff --git a/apps/locales/en_US/LC_MESSAGES/django.po b/apps/locales/en_US/LC_MESSAGES/django.po index 4f6fa10e3..b5e4c2e87 100644 --- a/apps/locales/en_US/LC_MESSAGES/django.po +++ b/apps/locales/en_US/LC_MESSAGES/django.po @@ -8648,4 +8648,13 @@ msgid "Multiple dialects, supporting 23 dialects" msgstr "" msgid "This interface is used to recognize short audio files within 60 seconds. Supports Mandarin Chinese, English, Cantonese, Japanese, Vietnamese, Malay, Indonesian, Filipino, Thai, Portuguese, Turkish, Arabic, Hindi, French, German, and 23 Chinese dialects." +msgstr "" + +msgid "CueWord" +msgstr "" + +msgid "If not passed, the default value is What is this audio saying? Only answer the audio content" +msgstr "" + +msgid "The Qwen Omni series model supports inputting multiple modalities of data, including video, audio, images, and text, and outputting audio and text." msgstr "" \ No newline at end of file diff --git a/apps/locales/zh_CN/LC_MESSAGES/django.po b/apps/locales/zh_CN/LC_MESSAGES/django.po index 62e97f258..6afeb1241 100644 --- a/apps/locales/zh_CN/LC_MESSAGES/django.po +++ b/apps/locales/zh_CN/LC_MESSAGES/django.po @@ -8774,4 +8774,13 @@ msgid "Multiple dialects, supporting 23 dialects" msgstr "多种方言，支持 23 种方言" msgid "This interface is used to recognize short audio files within 60 seconds. Supports Mandarin Chinese, English, Cantonese, Japanese, Vietnamese, Malay, Indonesian, Filipino, Thai, Portuguese, Turkish, Arabic, Hindi, French, German, and 23 Chinese dialects." -msgstr "本接口用于识别 60 秒之内的短音频文件。支持中文普通话、英语、粤语、日语、越南语、马来语、印度尼西亚语、菲律宾语、泰语、葡萄牙语、土耳其语、阿拉伯语、印地语、法语、德语及 23 种汉语方言。" \ No newline at end of file +msgstr "本接口用于识别 60 秒之内的短音频文件。支持中文普通话、英语、粤语、日语、越南语、马来语、印度尼西亚语、菲律宾语、泰语、葡萄牙语、土耳其语、阿拉伯语、印地语、法语、德语及 23 种汉语方言。" + +msgid "CueWord" +msgstr "提示词" + +msgid "If not passed, the default value is What is this audio saying? Only answer the audio content" +msgstr "如果未传递，默认值为 这段音频在说什么，只回答音频的内容" + +msgid "The Qwen Omni series model supports inputting multiple modalities of data, including video, audio, images, and text, and outputting audio and text." +msgstr "Qwen-Omni 系列模型支持输入多种模态的数据，包括视频、音频、图片、文本，并输出音频与文本" \ No newline at end of file diff --git a/apps/locales/zh_Hant/LC_MESSAGES/django.po b/apps/locales/zh_Hant/LC_MESSAGES/django.po index 794a0f859..6a5e91b6e 100644 --- a/apps/locales/zh_Hant/LC_MESSAGES/django.po +++ b/apps/locales/zh_Hant/LC_MESSAGES/django.po @@ -8774,4 +8774,13 @@ msgid "Multiple dialects, supporting 23 dialects" msgstr "多種方言，支持 23 種方言" msgid "This interface is used to recognize short audio files within 60 seconds. Supports Mandarin Chinese, English, Cantonese, Japanese, Vietnamese, Malay, Indonesian, Filipino, Thai, Portuguese, Turkish, Arabic, Hindi, French, German, and 23 Chinese dialects." -msgstr "本介面用於識別 60 秒之內的短音頻文件。支援中文普通話、英語、粵語、日語、越南語、馬來語、印度尼西亞語、菲律賓語、泰語、葡萄牙語、土耳其語、阿拉伯語、印地語、法語、德語及 23 種漢語方言。" \ No newline at end of file +msgstr "本介面用於識別 60 秒之內的短音頻文件。支援中文普通話、英語、粵語、日語、越南語、馬來語、印度尼西亞語、菲律賓語、泰語、葡萄牙語、土耳其語、阿拉伯語、印地語、法語、德語及 23 種漢語方言。" + +msgid "CueWord" +msgstr "提示詞" + +msgid "If not passed, the default value is What is this audio saying? Only answer the audio content" +msgstr "如果未傳遞，預設值為這段音訊在說什麼，只回答音訊的內容" + +msgid "The Qwen Omni series model supports inputting multiple modalities of data, including video, audio, images, and text, and outputting audio and text." +msgstr "Qwen-Omni系列模型支持輸入多種模態的數據，包括視頻、音訊、圖片、文字，並輸出音訊與文字" \ No newline at end of file diff --git a/apps/models_provider/impl/aliyun_bai_lian_model_provider/aliyun_bai_lian_model_provider.py b/apps/models_provider/impl/aliyun_bai_lian_model_provider/aliyun_bai_lian_model_provider.py index 6f27129be..56d570c04 100644 --- a/apps/models_provider/impl/aliyun_bai_lian_model_provider/aliyun_bai_lian_model_provider.py +++ b/apps/models_provider/impl/aliyun_bai_lian_model_provider/aliyun_bai_lian_model_provider.py @@ -15,6 +15,7 @@ from models_provider.impl.aliyun_bai_lian_model_provider.credential.embedding im AliyunBaiLianEmbeddingCredential from models_provider.impl.aliyun_bai_lian_model_provider.credential.image import QwenVLModelCredential from models_provider.impl.aliyun_bai_lian_model_provider.credential.llm import BaiLianLLMModelCredential +from models_provider.impl.aliyun_bai_lian_model_provider.credential.omi_stt import AliyunBaiLianOmiSTTModelCredential from models_provider.impl.aliyun_bai_lian_model_provider.credential.reranker import \ AliyunBaiLianRerankerCredential from models_provider.impl.aliyun_bai_lian_model_provider.credential.stt import AliyunBaiLianSTTModelCredential @@ -23,6 +24,7 @@ from models_provider.impl.aliyun_bai_lian_model_provider.credential.tts import A from models_provider.impl.aliyun_bai_lian_model_provider.model.embedding import AliyunBaiLianEmbedding from models_provider.impl.aliyun_bai_lian_model_provider.model.image import QwenVLChatModel from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel +from models_provider.impl.aliyun_bai_lian_model_provider.model.omi_stt import AliyunBaiLianOmiSpeechToText from models_provider.impl.aliyun_bai_lian_model_provider.model.reranker import AliyunBaiLianReranker from models_provider.impl.aliyun_bai_lian_model_provider.model.stt import AliyunBaiLianSpeechToText from models_provider.impl.aliyun_bai_lian_model_provider.model.tti import QwenTextToImageModel @@ -33,6 +35,7 @@ from django.utils.translation import gettext as _, gettext aliyun_bai_lian_model_credential = AliyunBaiLianRerankerCredential() aliyun_bai_lian_tts_model_credential = AliyunBaiLianTTSModelCredential() aliyun_bai_lian_stt_model_credential = AliyunBaiLianSTTModelCredential() +aliyun_bai_lian_omi_stt_model_credential = AliyunBaiLianOmiSTTModelCredential() aliyun_bai_lian_embedding_model_credential = AliyunBaiLianEmbeddingCredential() aliyun_bai_lian_llm_model_credential = BaiLianLLMModelCredential() qwenvl_model_credential = QwenVLModelCredential() @@ -73,7 +76,10 @@ model_info_list = [ModelInfo('gte-rerank', ModelInfo('qwen-plus', '', ModelTypeConst.LLM, aliyun_bai_lian_llm_model_credential, BaiLianChatModel), ModelInfo('qwen-max', '', ModelTypeConst.LLM, aliyun_bai_lian_llm_model_credential, - BaiLianChatModel) + BaiLianChatModel), + ModelInfo('qwen-omni-turbo', + _('The Qwen Omni series model supports inputting multiple modalities of data, including video, audio, images, and text, and outputting audio and text.'), + ModelTypeConst.STT, aliyun_bai_lian_omi_stt_model_credential, AliyunBaiLianOmiSpeechToText), ] module_info_vl_list = [ diff --git a/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/omi_stt.py b/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/omi_stt.py new file mode 100644 index 000000000..960b6e8ef --- /dev/null +++ b/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/omi_stt.py @@ -0,0 +1,72 @@ +# coding=utf-8 +import traceback +from typing import Dict, Any + +from common import forms +from common.exception.app_exception import AppApiException +from common.forms import BaseForm, PasswordInputField, TooltipLabel +from models_provider.base_model_provider import BaseModelCredential, ValidCode +from django.utils.translation import gettext as _ + +class AliyunBaiLianOmiSTTModelParams(BaseForm): + CueWord = forms.TextInputField( + TooltipLabel(_('CueWord'), _('If not passed, the default value is What is this audio saying? Only answer the audio content')), + required=True, + default_value='这段音频在说什么，只回答音频的内容', + ) + + +class AliyunBaiLianOmiSTTModelCredential(BaseForm, BaseModelCredential): + api_key = PasswordInputField("API key", required=True) + + def is_valid(self, + model_type: str, + model_name: str, + model_credential: Dict[str, Any], + model_params: Dict[str, Any], + provider, + raise_exception: bool = False + ) -> bool: + + model_type_list = provider.get_model_type_list() + if not any(mt.get('value') == model_type for mt in model_type_list): + raise AppApiException( + ValidCode.valid_error.value, + _('{model_type} Model type is not supported').format(model_type=model_type) + ) + + required_keys = ['api_key'] + for key in required_keys: + if key not in model_credential: + if raise_exception: + raise AppApiException( + ValidCode.valid_error.value, + _('{key} is required').format(key=key) + ) + return False + + try: + model = provider.get_model(model_type, model_name, model_credential) + except Exception as e: + traceback.print_exc() + if isinstance(e, AppApiException): + raise e + if raise_exception: + raise AppApiException( + ValidCode.valid_error.value, + _('Verification failed, please check whether the parameters are correct: {error}').format(error=str(e)) + ) + return False + return True + + def encryption_dict(self, model: Dict[str, object]) -> Dict[str, object]: + + return { + **model, + 'api_key': super().encryption(model.get('api_key', '')) + } + + + def get_model_params_setting_form(self, model_name): + + return AliyunBaiLianOmiSTTModelParams() diff --git a/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/omi_stt.py b/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/omi_stt.py new file mode 100644 index 000000000..4528f1cef --- /dev/null +++ b/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/omi_stt.py @@ -0,0 +1,87 @@ +import base64 +import os +import traceback +from typing import Dict + +from openai import OpenAI + +from common.utils.logger import maxkb_logger +from models_provider.base_model_provider import MaxKBBaseModel +from models_provider.impl.base_stt import BaseSpeechToText + + +class AliyunBaiLianOmiSpeechToText(MaxKBBaseModel, BaseSpeechToText): + api_key: str + model: str + params: dict + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.api_key = kwargs.get('api_key') + self.model = kwargs.get('model') + self.params = kwargs.get('params') + + @staticmethod + def is_cache_model(): + return False + + @staticmethod + def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs): + return AliyunBaiLianOmiSpeechToText( + model=model_name, + api_key=model_credential.get('api_key'), + params= model_kwargs, + **model_kwargs + ) + + + def check_auth(self): + cwd = os.path.dirname(os.path.abspath(__file__)) + with open(f'{cwd}/iat_mp3_16k.mp3', 'rb') as audio_file: + self.speech_to_text(audio_file) + + + + def speech_to_text(self, audio_file): + try: + client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + api_key=self.api_key, + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", + ) + + base64_audio = base64.b64encode(audio_file.read()).decode("utf-8") + + completion = client.chat.completions.create( + model="qwen-omni-turbo-0119", + messages=[ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": f"data:;base64,{base64_audio}", + "format": "mp3", + }, + }, + {"type": "text", "text": self.params.get('CueWord')}, + ], + }, + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text"], + audio={"voice": "Cherry", "format": "mp3"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, + ) + result = [] + for chunk in completion: + if chunk.choices and hasattr(chunk.choices[0].delta, 'audio'): + transcript = chunk.choices[0].delta.audio.get('transcript') + result.append(transcript) + return "".join(result) + + except Exception as err: + maxkb_logger.error(f":Error: {str(err)}: {traceback.format_exc()}") diff --git a/apps/models_provider/impl/tencent_model_provider/model/stt.py b/apps/models_provider/impl/tencent_model_provider/model/stt.py index 3b5f050b0..a501fed19 100644 --- a/apps/models_provider/impl/tencent_model_provider/model/stt.py +++ b/apps/models_provider/impl/tencent_model_provider/model/stt.py @@ -1,6 +1,7 @@ import base64 import json import os +import traceback from typing import Dict from tencentcloud.asr.v20190614 import asr_client, models @@ -9,6 +10,7 @@ from tencentcloud.common.exception import TencentCloudSDKException from tencentcloud.common.profile.client_profile import ClientProfile from tencentcloud.common.profile.http_profile import HttpProfile +from common.utils.logger import maxkb_logger from models_provider.base_model_provider import MaxKBBaseModel from models_provider.impl.base_stt import BaseSpeechToText