15308555518/ MaxKB
1
0
Fork 0
mirror of https://github.com/1Panel-dev/MaxKB.git synced 2025-12-30 01:32:49 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge branch 'main' into pr@main@application_flow

This commit is contained in:
wangdan-fit2cloud 2024-06-05 10:09:01 +08:00
parent b45a82c519 ac25c39280
commit 7b88a09a22
23 changed files with 127 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/application/serializers/application_serializers.py
View File

@ -317,9 +317,9 @@ class ApplicationSerializer(serializers.Serializer):
id = serializers.CharField(required=True, error_messages=ErrMessage.uuid("应用id"))
user_id = serializers.UUIDField(required=False, error_messages=ErrMessage.uuid("用户id"))
query_text = serializers.CharField(required=True, error_messages=ErrMessage.char("查询文本"))
top_number = serializers.IntegerField(required=True, max_value=10, min_value=1,
top_number = serializers.IntegerField(required=True, max_value=100, min_value=1,
error_messages=ErrMessage.integer("topN"))
similarity = serializers.FloatField(required=True, max_value=1, min_value=0,
similarity = serializers.FloatField(required=True, max_value=2, min_value=0,
error_messages=ErrMessage.float("相关度"))
search_mode = serializers.CharField(required=True, validators=[
validators.RegexValidator(regex=re.compile("^embedding|keywords|blend$"),

4
apps/application/serializers/chat_serializers.py
View File

@ -422,11 +422,11 @@ class ChatRecordSerializer(serializers.Serializer):
return True
class ImproveSerializer(serializers.Serializer):
title = serializers.CharField(required=False, allow_null=True, allow_blank=True,
title = serializers.CharField(required=False, max_length=256, allow_null=True, allow_blank=True,
error_messages=ErrMessage.char("段落标题"))
content = serializers.CharField(required=True, error_messages=ErrMessage.char("段落内容"))
problem_text = serializers.CharField(required=False, allow_null=True, allow_blank=True,
problem_text = serializers.CharField(required=False, max_length=256, allow_null=True, allow_blank=True,
error_messages=ErrMessage.char("问题"))
class ParagraphModel(serializers.ModelSerializer):

4
apps/common/event/listener_manage.py
View File

@ -6,6 +6,7 @@
@date：2023/10/20 14:01
@desc:
"""
import datetime
import logging
import os
import traceback
@ -143,7 +144,8 @@ class ListenerManagement:
status = Status.error
finally:
# 修改状态
QuerySet(Document).filter(id=document_id).update(**{'status': status})
QuerySet(Document).filter(id=document_id).update(
**{'status': status, 'update_time': datetime.datetime.now()})
QuerySet(Paragraph).filter(document_id=document_id).update(**{'status': status})
max_kb.info(f"结束--->向量化文档:{document_id}")

5
apps/common/handle/impl/html_split_handle.py
View File

@ -37,14 +37,9 @@ def get_encoding(buffer):
class HTMLSplitHandle(BaseSplitHandle):
def support(self, file, get_buffer):
buffer = get_buffer(file)
file_name: str = file.name.lower()
if file_name.endswith(".html"):
return True
result = detect(buffer)
if result['encoding'] is not None and result['confidence'] is not None and result['encoding'] != 'ascii' and \
result['confidence'] > 0.5:
return True
return False
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):

2
apps/common/util/split_model.py
View File

@ -163,7 +163,7 @@ def parse_level(text, pattern: str):
:param pattern: 正则
:return: 符合正则的文本
"""
level_content_list = list(map(to_tree_obj, re_findall(pattern, text)))
level_content_list = list(map(to_tree_obj, [r[0:255] for r in re_findall(pattern, text) if r is not None]))
return list(map(filter_special_symbol, level_content_list))

4
apps/dataset/serializers/dataset_serializers.py
View File

@ -535,9 +535,9 @@ class DataSetSerializers(serializers.ModelSerializer):
id = serializers.CharField(required=True, error_messages=ErrMessage.char("id"))
user_id = serializers.UUIDField(required=False, error_messages=ErrMessage.char("用户id"))
query_text = serializers.CharField(required=True, error_messages=ErrMessage.char("查询文本"))
top_number = serializers.IntegerField(required=True, max_value=10, min_value=1,
top_number = serializers.IntegerField(required=True, max_value=100, min_value=1,
error_messages=ErrMessage.char("响应Top"))
similarity = serializers.FloatField(required=True, max_value=1, min_value=0,
similarity = serializers.FloatField(required=True, max_value=2, min_value=0,
error_messages=ErrMessage.char("相似度"))
search_mode = serializers.CharField(required=True, validators=[
validators.RegexValidator(regex=re.compile("^embedding|keywords|blend$"),

65
apps/smartdoc/conf.py
View File

@ -13,6 +13,7 @@ import os
import re
from importlib import import_module
from urllib.parse import urljoin, urlparse
import yaml
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@ -75,25 +76,18 @@ class DoesNotExist(Exception):
class Config(dict):
defaults = {
# 数据库相关配置
"DB_HOST": "",
"DB_PORT": "",
"DB_USER": "",
"DB_PASSWORD": "",
"DB_HOST": "127.0.0.1",
"DB_PORT": 5432,
"DB_USER": "root",
"DB_PASSWORD": "Password123@postgres",
"DB_ENGINE": "django.db.backends.postgresql_psycopg2",
# 邮件相关配置
"EMAIL_ADDRESS": "",
"EMAIL_USE_TLS": False,
"EMAIL_USE_SSL": True,
"EMAIL_HOST": "",
"EMAIL_PORT": 465,
"EMAIL_HOST_USER": "",
"EMAIL_HOST_PASSWORD": "",
# 向量模型
"EMBEDDING_MODEL_NAME": "shibing624/text2vec-base-chinese",
"EMBEDDING_DEVICE": "cpu",
"EMBEDDING_MODEL_PATH": os.path.join(PROJECT_DIR, 'models'),
# 向量库配置
"VECTOR_STORE_NAME": 'pg_vector'
"VECTOR_STORE_NAME": 'pg_vector',
"DEBUG": False
}
@ -180,8 +174,36 @@ class ConfigManager:
loaded = self.from_yaml(i)
if loaded:
return True
msg = f"""
return False
Error: No config file found.
You can run `cp config_example.yml {self.root_path}/config.yml`, and edit it.
"""
raise ImportError(msg)
def load_from_env(self):
keys = os.environ.keys()
config = {key.replace('MAXKB_', ''): os.environ.get(key) for key in keys if key.startswith('MAXKB_')}
if len(config.keys()) <= 1:
msg = f"""
Error: No config env found.
Please set environment variables
MAXKB_CONFIG_TYPE: 配置文件读取方式 FILE: 使用配置文件配置 ENV: 使用ENV配置
MAXKB_DB_NAME: 数据库名称
MAXKB_DB_HOST: 数据库主机
MAXKB_DB_PORT: 数据库端口
MAXKB_DB_USER: 数据库用户名
MAXKB_DB_PASSWORD: 数据库密码
MAXKB_EMBEDDING_MODEL_PATH: 向量模型目录
MAXKB_EMBEDDING_MODEL_NAME: 向量模型名称
"""
raise ImportError(msg)
self.from_mapping(config)
return True
@classmethod
def load_user_config(cls, root_path=None, config_class=None):
@ -190,15 +212,10 @@ class ConfigManager:
if not root_path:
root_path = PROJECT_DIR
manager = cls(root_path=root_path)
if manager.load_from_yml():
config = manager.config
config_type = os.environ.get('MAXKB_CONFIG_TYPE')
if config_type is None or config_type != 'ENV':
manager.load_from_yml()
else:
msg = f"""
Error: No config file found.
You can run `cp config_example.yml {root_path}/config.yml`, and edit it.
"""
raise ImportError(msg)
manager.load_from_env()
config = manager.config
return config

9
config_example.yml
View File

@ -1,12 +1,3 @@
# 邮箱配置
EMAIL_ADDRESS:
EMAIL_USE_TLS: False
EMAIL_USE_SSL: True
EMAIL_HOST: smtp.qq.com
EMAIL_PORT: 465
EMAIL_HOST_USER:
EMAIL_HOST_PASSWORD:
# 数据库链接信息
DB_NAME: maxkb
DB_HOST: localhost

12
installer/Dockerfile
View File

@ -17,7 +17,6 @@ RUN apt-get update && \
COPY . /opt/maxkb/app
RUN mkdir -p /opt/maxkb/app /opt/maxkb/model /opt/maxkb/conf && \
cp -f /opt/maxkb/app/installer/config.yaml /opt/maxkb/conf && \
rm -rf /opt/maxkb/app/ui
COPY --from=web-build ui /opt/maxkb/app/ui
WORKDIR /opt/maxkb/app
@ -33,7 +32,16 @@ ARG DOCKER_IMAGE_TAG=dev \
BUILD_AT \
GITHUB_COMMIT
ENV MAXKB_VERSION ${DOCKER_IMAGE_TAG} (build at ${BUILD_AT}, commit: ${GITHUB_COMMIT})
ENV MAXKB_VERSION="${DOCKER_IMAGE_TAG} (build at ${BUILD_AT}, commit: ${GITHUB_COMMIT})" \
MAXKB_CONFIG_TYPE=ENV \
MAXKB_DB_NAME=maxkb \
MAXKB_DB_HOST=127.0.0.1 \
MAXKB_DB_PORT=5432 \
MAXKB_DB_USER=root \
MAXKB_DB_PASSWORD=Password123@postgres \
MAXKB_EMBEDDING_MODEL_NAME=/opt/maxkb/model/embedding/shibing624_text2vec-base-chinese \
MAXKB_EMBEDDING_MODEL_PATH=/opt/maxkb/model/embedding
WORKDIR /opt/maxkb/app
COPY --from=stage-build /opt/maxkb /opt/maxkb
COPY --from=stage-build /opt/py3 /opt/py3

6
ui/src/assets/html-icon.svg Normal file
View File

@ -0,0 +1,6 @@
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M4.00024 2.5C4.00024 1.94772 4.44796 1.5 5.00024 1.5H14.7931C14.9257 1.5 15.0529 1.55268 15.1467 1.64645L19.8538 6.35355C19.9476 6.44732 20.0002 6.5745 20.0002 6.70711V21.5C20.0002 22.0523 19.5525 22.5 19.0002 22.5H5.00024C4.44796 22.5 4.00024 22.0523 4.00024 21.5V2.5Z" fill="#D136D1"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M4.00024 2.5C4.00024 1.94772 4.44796 1.5 5.00024 1.5H14.7931C14.9257 1.5 15.0529 1.55268 15.1467 1.64645L19.8538 6.35355C19.9476 6.44732 20.0002 6.5745 20.0002 6.70711V21.5C20.0002 22.0523 19.5525 22.5 19.0002 22.5H5.00024C4.44796 22.5 4.00024 22.0523 4.00024 21.5V2.5Z" fill="#3370FF"/>
<path d="M10.2966 11.4764L7.76958 14.1363L10.2966 16.7961C10.3873 16.8916 10.3857 17.0447 10.293 17.1381L10.292 17.1391L9.95252 17.4774C9.8597 17.5698 9.7118 17.5677 9.62149 17.4727L6.61239 14.3054C6.52308 14.2114 6.52308 14.0611 6.61239 13.9671L9.62149 10.7999C9.7118 10.7048 9.8597 10.7027 9.95252 10.7952L10.292 11.1335C10.3852 11.2263 10.3877 11.3794 10.2976 11.4754L10.2966 11.4764ZM16.3178 14.1363L13.9712 11.4764C13.887 11.381 13.8885 11.2278 13.9746 11.1344L13.9755 11.1335L14.2908 10.7952C14.3769 10.7027 14.5143 10.7048 14.5981 10.7999L17.3923 13.9671C17.4752 14.0611 17.4752 14.2114 17.3923 14.3054L14.5981 17.4727C14.5143 17.5677 14.3769 17.5698 14.2908 17.4774L13.9755 17.1391C13.8889 17.0462 13.8866 16.8931 13.9704 16.7971L13.9712 16.7961L16.3178 14.1363ZM12.6285 9.09234L13.1203 9.14509C13.2546 9.15949 13.3509 9.27213 13.3353 9.39669L12.1614 18.7083C12.1457 18.8327 12.0244 18.9219 11.8902 18.9075L11.3984 18.8547C11.2642 18.8403 11.1679 18.7277 11.1834 18.6031L12.3574 9.2915C12.373 9.16708 12.4944 9.07796 12.6285 9.09234Z" fill="white"/>
<path d="M15 1.54492C15.054 1.56949 15.1037 1.6037 15.1464 1.64646L19.8536 6.35357C19.8963 6.39632 19.9305 6.44602 19.9551 6.50001H16C15.4477 6.50001 15 6.0523 15 5.50001V1.54492Z" fill="#2B5FD9"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.0 KiB

7
ui/src/components/ai-chat/OperationButton.vue
View File

@ -5,8 +5,8 @@
</el-text>
</div>
<div>
<el-tooltip effect="dark" content="重新生成" placement="top">
<el-button text @click="regeneration">
<el-tooltip effect="dark" content="换个答案" placement="top">
<el-button :disabled="chat_loading" text @click="regeneration">
<AppIcon iconName="VideoPlay"></AppIcon>
</el-button>
</el-tooltip>
@ -78,6 +78,9 @@ const props = defineProps({
type: String,
default: ''
},
chat_loading: {
type: Boolean
},
log: Boolean
})

5
ui/src/components/ai-chat/index.vue
View File

@ -131,6 +131,7 @@
:data="item"
:applicationId="appId"
:chatId="chartOpenId"
:chat_loading="loading"
@regeneration="regenerationChart(item)"
/>
</div>
@ -541,7 +542,9 @@ function chatMessage(chat?: any, problem?: string, re_chat?: boolean) {
function regenerationChart(item: chatType) {
inputValue.value = item.problem_text
chatMessage(null, '', true)
if (!loading.value) {
chatMessage(null, '', true)
}
}
function getSourceDetail(row: any) {

12
ui/src/layout/hooks/useResize.ts
View File

@ -1,4 +1,4 @@
import { watch, onBeforeMount, onMounted, onBeforeUnmount } from 'vue'
import { nextTick, onBeforeMount, onMounted, onBeforeUnmount } from 'vue'
import { useRoute } from 'vue-router'
import useStore from '@/stores'
import { DeviceType } from '@/enums/common'
@ -9,7 +9,7 @@ const WIDTH = 600
export default () => {
const { common } = useStore()
const _isMobile = () => {
const rect = document.body.getBoundingClientRect()
const rect = document.body?.getBoundingClientRect()
return rect.width - 1 < WIDTH
}
@ -25,9 +25,11 @@ export default () => {
})
onMounted(() => {
if (_isMobile()) {
common.toggleDevice(DeviceType.Mobile)
}
nextTick(() => {
if (_isMobile()) {
common.toggleDevice(DeviceType.Mobile)
}
})
})
onBeforeUnmount(() => {

2
ui/src/locales/lang/en_US/views/application-overview.ts
View File

@ -61,7 +61,7 @@ export default {
LimitDialog: {
dialogTitle: 'Access Restrictions',
showSourceLabel: 'Show Source',
clientQueryLimitLabel: 'Client Query Limit',
clientQueryLimitLabel: 'Each Client Query Limit',
timesDays: 'Times/Day',
whitelistLabel: 'Whitelist',
whitelistPlaceholder: 'Please enter allowed third-party source addresses, one per line, such as:\nhttp://127.0.0.1:5678\nhttps://dataease.io',

2
ui/src/locales/lang/zh_CN/views/application-overview.ts
View File

@ -61,7 +61,7 @@ export default {
LimitDialog:{
dialogTitle: '访问限制',
showSourceLabel: '显示知识来源',
clientQueryLimitLabel: '客户端提问限制',
clientQueryLimitLabel: '每个客户端提问限制',
timesDays: '次/天',
whitelistLabel: '白名单',
whitelistPlaceholder: '请输入允许嵌入第三方的源地址，一行一个，如：\nhttp://127.0.0.1:5678\nhttps://dataease.io',

10
ui/src/stores/modules/application.ts
View File

@ -65,14 +65,8 @@ const useApplicationStore = defineStore({
applicationApi
.postAppAuthentication(token, loading)
.then((res) => {
const accessTokenObjStr = localStorage.getItem('accessTokenObj')
if (accessTokenObjStr) {
const accessTokenObj = JSON.parse(accessTokenObjStr)
accessTokenObj[token] = res.data
localStorage.setItem('accessTokenObj', JSON.stringify(accessTokenObj))
} else {
localStorage.setItem('accessTokenObj', JSON.stringify({ [token]: res.data }))
}
localStorage.setItem('accessToken', res.data)
sessionStorage.setItem('accessToken', res.data)
resolve(res)
})
.catch((error) => {

16
ui/src/stores/modules/user.ts
View File

@ -16,8 +16,7 @@ const useUserStore = defineStore({
userType: 1,
userInfo: null,
token: '',
version: '',
accessToken: ''
version: ''
}),
actions: {
getToken(): String | null {
@ -27,13 +26,9 @@ const useUserStore = defineStore({
return this.userType === 1 ? localStorage.getItem('token') : this.getAccessToken()
},
getAccessToken() {
const accessTokenObjStr = localStorage.getItem('accessTokenObj')
if (accessTokenObjStr && this.accessToken) {
const accessTokenObj = JSON.parse(accessTokenObjStr)
const result = accessTokenObj[this.accessToken]
if (result) {
return result
}
const accessToken = sessionStorage.getItem('accessToken')
if (accessToken) {
return accessToken
}
return localStorage.getItem('accessToken')
},
@ -55,9 +50,6 @@ const useUserStore = defineStore({
changeUserType(num: number) {
this.userType = num
},
setAccessToken(accessToken: string) {
this.accessToken = accessToken
},
async asyncGetVersion() {
return UserApi.getVersion().then((ok) => {

1
ui/src/views/chat/base/index.vue
View File

@ -54,7 +54,6 @@ function getProfile() {
onMounted(() => {
user.changeUserType(2)
user.setAccessToken(accessToken)
getAccessToken(accessToken)
})
</script>

15
ui/src/views/chat/embed/index.vue
View File

@ -15,7 +15,9 @@
:chatId="currentChatId"
@refresh="refresh"
@scroll="handleScroll"
></AiChat>
class="AiChat-embed"
>
</AiChat>
</div>
<el-button type="primary" link class="new-chat-button" @click="newChat">
@ -194,7 +196,6 @@ function refresh(id: string) {
onMounted(() => {
user.changeUserType(2)
user.setAccessToken(accessToken)
getAccessToken(accessToken)
})
</script>
@ -220,7 +221,7 @@ onMounted(() => {
}
.new-chat-button {
position: absolute;
bottom: 84px;
bottom: 80px;
left: 18px;
z-index: 11;
}
@ -277,5 +278,13 @@ onMounted(() => {
max-width: var(--app-chat-width, 860px);
margin: 0 auto;
}
.AiChat-embed {
.ai-chat__operate {
padding-top: 38px;
}
.ai-chat__content {
padding-bottom: 104px
}
}
}
</style>

5
ui/src/views/chat/pc/index.vue
View File

@ -59,9 +59,7 @@
{{ paginationConfig.total }} 条提问
</span>
<el-dropdown class="ml-8">
<el-tooltip effect="dark" content="导出聊天记录" placement="top">
<AppIcon iconName="app-export" class="cursor"></AppIcon>
</el-tooltip>
<AppIcon iconName="app-export" class="cursor" title="导出聊天记录"></AppIcon>
<template #dropdown>
<el-dropdown-menu>
<el-dropdown-item @click="exportMarkdown">导出 Markdown</el-dropdown-item>
@ -286,7 +284,6 @@ async function exportHTML(): Promise<void> {
onMounted(() => {
user.changeUserType(2)
user.setAccessToken(accessToken)
getAccessToken(accessToken)
})
</script>

2
ui/src/views/hit-test/index.vue
View File

@ -144,7 +144,7 @@
<el-input-number
v-model="cloneForm.top_number"
:min="1"
:max="10"
:max="100"
controls-position="right"
class="w-full"
/>

15
ui/src/views/log/component/EditContentDialog.vue
View File

@ -9,7 +9,13 @@
@submit.prevent
>
<el-form-item label="关联问题">
<el-input v-model="form.problem_text" placeholder="关联问题"> </el-input>
<el-input
v-model="form.problem_text"
placeholder="关联问题"
maxlength="256"
show-word-limit
>
</el-input>
</el-form-item>
<el-form-item label="内容" prop="content">
<el-input
@ -23,7 +29,12 @@
</el-input>
</el-form-item>
<el-form-item label="标题">
<el-input v-model="form.title" placeholder="请给当前内容设置一个标题，以便管理查看">
<el-input
show-word-limit
v-model="form.title"
placeholder="请给当前内容设置一个标题，以便管理查看"
maxlength="256"
>
</el-input>
</el-form-item>
<el-form-item label="选择知识库" prop="dataset_id">

11
ui/src/views/paragraph/component/ParagraphForm.vue
View File

@ -8,7 +8,14 @@
@submit.prevent
>
<el-form-item label="分段标题">
<el-input v-if="isEdit" v-model="form.title" placeholder="请输入分段标题"> </el-input>
<el-input
v-if="isEdit"
v-model="form.title"
placeholder="请输入分段标题"
maxlength="256"
show-word-limit
>
</el-input>
<span class="lighter" v-else>{{ form.title || '-' }}</span>
</el-form-item>
<el-form-item label="分段内容" prop="content">
@ -24,7 +31,7 @@
:footers="footers"
>
<template #defFooters>
<span style="margin-left: -6px;">/ 4096</span>
<span style="margin-left: -6px">/ 4096</span>
</template>
</MarkdownEditor>
<MdPreview