mirror of
https://github.com/1Panel-dev/MaxKB.git
synced 2025-12-28 05:42:51 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
30b1bdfe5e | ||
|
|
d1e9081a43 |
|
|
@ -46,14 +46,14 @@ class OpenAIChatModel(MaxKBBaseModel, ChatOpenAI):
|
||||||
|
|
||||||
def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:
|
def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:
|
||||||
try:
|
try:
|
||||||
super().get_num_tokens_from_messages(messages)
|
return super().get_num_tokens_from_messages(messages)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
tokenizer = TokenizerManage.get_tokenizer()
|
tokenizer = TokenizerManage.get_tokenizer()
|
||||||
return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages])
|
return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages])
|
||||||
|
|
||||||
def get_num_tokens(self, text: str) -> int:
|
def get_num_tokens(self, text: str) -> int:
|
||||||
try:
|
try:
|
||||||
super().get_num_tokens(text)
|
return super().get_num_tokens(text)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
tokenizer = TokenizerManage.get_tokenizer()
|
tokenizer = TokenizerManage.get_tokenizer()
|
||||||
return len(tokenizer.encode(text))
|
return len(tokenizer.encode(text))
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,7 @@ django-apscheduler = "^0.6.2"
|
||||||
pymupdf = "1.24.9"
|
pymupdf = "1.24.9"
|
||||||
pypdf = "4.3.1"
|
pypdf = "4.3.1"
|
||||||
rapidocr-onnxruntime = "1.3.24"
|
rapidocr-onnxruntime = "1.3.24"
|
||||||
|
opencv-python-headless = "^4.10.0"
|
||||||
python-docx = "^1.1.0"
|
python-docx = "^1.1.0"
|
||||||
xlwt = "^1.3.0"
|
xlwt = "^1.3.0"
|
||||||
dashscope = "^1.17.0"
|
dashscope = "^1.17.0"
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue