fix: 修复openai计算token错误

(cherry picked from commit 3d1b3ea8d5)
ci: 增加辅助pip
2025-12-26 01:33:05 +00:00 · 2024-08-29 17:24:00 +08:00 · 2024-08-29 17:23:49 +08:00
2 changed files with 3 additions and 2 deletions
--- a/apps/setting/models_provider/impl/openai_model_provider/model/llm.py
+++ b/apps/setting/models_provider/impl/openai_model_provider/model/llm.py
@ -46,14 +46,14 @@ class OpenAIChatModel(MaxKBBaseModel, ChatOpenAI):

    def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:
        try:
-            super().get_num_tokens_from_messages(messages)
+            return super().get_num_tokens_from_messages(messages)
        except Exception as e:
            tokenizer = TokenizerManage.get_tokenizer()
            return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages])

    def get_num_tokens(self, text: str) -> int:
        try:
-            super().get_num_tokens(text)
+            return super().get_num_tokens(text)
        except Exception as e:
            tokenizer = TokenizerManage.get_tokenizer()
            return len(tokenizer.encode(text))
--- a/pyproject.toml
+++ b/pyproject.toml
@ -33,6 +33,7 @@ django-apscheduler = "^0.6.2"
 pymupdf = "1.24.9"
 pypdf = "4.3.1"
 rapidocr-onnxruntime = "1.3.24"
+opencv-python-headless = "^4.10.0"
 python-docx = "^1.1.0"
 xlwt = "^1.3.0"
 dashscope = "^1.17.0"
Author	SHA1	Message	Date
shaohuzhang1	30b1bdfe5e	fix: 修复openai计算token错误 Some checks failed sync2gitee / repo-sync (push) Has been cancelled Details Typos Check / Spell Check with Typos (push) Has been cancelled Details (cherry picked from commit `3d1b3ea8d5`)	2024-08-29 17:24:00 +08:00
CaptainB	d1e9081a43	ci: 增加辅助pip (cherry picked from commit `b0931c8612`)	2024-08-29 17:23:49 +08:00