diff --git a/apps/setting/models_provider/impl/ollama_model_provider/credential/image.py b/apps/setting/models_provider/impl/ollama_model_provider/credential/image.py new file mode 100644 index 000000000..c7077138f --- /dev/null +++ b/apps/setting/models_provider/impl/ollama_model_provider/credential/image.py @@ -0,0 +1,65 @@ +# coding=utf-8 +import base64 +import os +from typing import Dict + +from langchain_core.messages import HumanMessage + +from common import forms +from common.exception.app_exception import AppApiException +from common.forms import BaseForm, TooltipLabel +from setting.models_provider.base_model_provider import BaseModelCredential, ValidCode + +class OllamaImageModelParams(BaseForm): + temperature = forms.SliderField(TooltipLabel('温度', '较高的数值会使输出更加随机，而较低的数值会使其更加集中和确定'), + required=True, default_value=0.7, + _min=0.1, + _max=1.0, + _step=0.01, + precision=2) + + max_tokens = forms.SliderField( + TooltipLabel('输出最大Tokens', '指定模型可生成的最大token个数'), + required=True, default_value=800, + _min=1, + _max=100000, + _step=1, + precision=0) + + + +class OllamaImageModelCredential(BaseForm, BaseModelCredential): + api_base = forms.TextInputField('API 域名', required=True) + api_key = forms.PasswordInputField('API Key', required=True) + + def is_valid(self, model_type: str, model_name, model_credential: Dict[str, object], provider, + raise_exception=False): + model_type_list = provider.get_model_type_list() + if not any(list(filter(lambda mt: mt.get('value') == model_type, model_type_list))): + raise AppApiException(ValidCode.valid_error.value, f'{model_type} 模型类型不支持') + + for key in ['api_base', 'api_key']: + if key not in model_credential: + if raise_exception: + raise AppApiException(ValidCode.valid_error.value, f'{key} 字段为必填字段') + else: + return False + try: + model = provider.get_model(model_type, model_name, model_credential) + res = model.stream([HumanMessage(content=[{"type": "text", "text": "你好"}])]) + for chunk in res: + print(chunk) + except Exception as e: + if isinstance(e, AppApiException): + raise e + if raise_exception: + raise AppApiException(ValidCode.valid_error.value, f'校验失败,请检查参数是否正确: {str(e)}') + else: + return False + return True + + def encryption_dict(self, model: Dict[str, object]): + return {**model, 'api_key': super().encryption(model.get('api_key', ''))} + + def get_model_params_setting_form(self, model_name): + return OllamaImageModelParams() diff --git a/apps/setting/models_provider/impl/ollama_model_provider/model/image.py b/apps/setting/models_provider/impl/ollama_model_provider/model/image.py new file mode 100644 index 000000000..82c8df92e --- /dev/null +++ b/apps/setting/models_provider/impl/ollama_model_provider/model/image.py @@ -0,0 +1,26 @@ +from typing import Dict + +from langchain_openai.chat_models import ChatOpenAI + +from common.config.tokenizer_manage_config import TokenizerManage +from setting.models_provider.base_model_provider import MaxKBBaseModel + + +def custom_get_token_ids(text: str): + tokenizer = TokenizerManage.get_tokenizer() + return tokenizer.encode(text) + + +class OllamaImage(MaxKBBaseModel, ChatOpenAI): + + @staticmethod + def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs): + optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs) + return OllamaImage( + model_name=model_name, + openai_api_base=model_credential.get('api_base'), + openai_api_key=model_credential.get('api_key'), + # stream_options={"include_usage": True}, + streaming=True, + **optional_params, + ) diff --git a/apps/setting/models_provider/impl/ollama_model_provider/ollama_model_provider.py b/apps/setting/models_provider/impl/ollama_model_provider/ollama_model_provider.py index a69083010..bd3f9c8b2 100644 --- a/apps/setting/models_provider/impl/ollama_model_provider/ollama_model_provider.py +++ b/apps/setting/models_provider/impl/ollama_model_provider/ollama_model_provider.py @@ -21,8 +21,10 @@ from common.util.file_util import get_file_content from setting.models_provider.base_model_provider import IModelProvider, ModelProvideInfo, ModelInfo, ModelTypeConst, \ BaseModelCredential, DownModelChunk, DownModelChunkStatus, ValidCode, ModelInfoManage from setting.models_provider.impl.ollama_model_provider.credential.embedding import OllamaEmbeddingModelCredential +from setting.models_provider.impl.ollama_model_provider.credential.image import OllamaImageModelCredential from setting.models_provider.impl.ollama_model_provider.credential.llm import OllamaLLMModelCredential from setting.models_provider.impl.ollama_model_provider.model.embedding import OllamaEmbedding +from setting.models_provider.impl.ollama_model_provider.model.image import OllamaImage from setting.models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel from smartdoc.conf import PROJECT_DIR @@ -133,6 +135,7 @@ model_info_list = [ ModelTypeConst.LLM, ollama_llm_model_credential, OllamaChatModel), ] ollama_embedding_model_credential = OllamaEmbeddingModelCredential() +ollama_image_model_credential = OllamaImageModelCredential() embedding_model_info = [ ModelInfo( 'nomic-embed-text', @@ -140,15 +143,36 @@ embedding_model_info = [ ModelTypeConst.EMBEDDING, ollama_embedding_model_credential, OllamaEmbedding), ] -model_info_manage = ModelInfoManage.builder().append_model_info_list(model_info_list).append_model_info_list( - embedding_model_info).append_default_model_info( +image_model_info = [ ModelInfo( + 'llava:7b', + '', + ModelTypeConst.IMAGE, ollama_image_model_credential, OllamaImage), + ModelInfo( + 'llava:13b', + '', + ModelTypeConst.IMAGE, ollama_image_model_credential, OllamaImage), + ModelInfo( + 'llava:34b', + '', + ModelTypeConst.IMAGE, ollama_image_model_credential, OllamaImage), +] + +model_info_manage = ( + ModelInfoManage.builder() + .append_model_info_list(model_info_list) + .append_model_info_list(embedding_model_info) + .append_default_model_info(ModelInfo( 'phi3', 'Phi-3 Mini是Microsoft的3.8B参数，轻量级，最先进的开放模型。', - ModelTypeConst.LLM, ollama_llm_model_credential, OllamaChatModel)).append_default_model_info(ModelInfo( - 'nomic-embed-text', - '一个具有大令牌上下文窗口的高性能开放嵌入模型。', - ModelTypeConst.EMBEDDING, ollama_embedding_model_credential, OllamaEmbedding), ).build() + ModelTypeConst.LLM, ollama_llm_model_credential, OllamaChatModel)) + .append_default_model_info(ModelInfo( + 'nomic-embed-text', + '一个具有大令牌上下文窗口的高性能开放嵌入模型。', + ModelTypeConst.EMBEDDING, ollama_embedding_model_credential, OllamaEmbedding), ) + .append_model_info_list(image_model_info) + .build() +) def get_base_url(url: str):