15308555518/ MaxKB
1
0
Fork 0
mirror of https://github.com/1Panel-dev/MaxKB.git synced 2025-12-28 14:52:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/maxkb-dev/maxkb

This commit is contained in:
liqiang-fit2cloud 2024-11-27 14:10:17 +08:00
parent c0a04eee6e b7905198a2
commit 58c0c34b2b
26 changed files with 303 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/application/flow/step_node/document_extract_node/impl/base_document_extract_node.py
View File

@ -29,18 +29,20 @@ class BaseDocumentExtractNode(IDocumentExtractNode):
# 回到文件头
buffer.seek(0)
file_content = split_handle.get_content(buffer)
content.append( '## ' + doc['name'] + '\n' + file_content)
content.append('## ' + doc['name'] + '\n' + file_content)
break
return NodeResult({'content': splitter.join(content)}, {})
def get_details(self, index: int, **kwargs):
# 不保存content全部内容，因为content内容可能会很大
content = (self.context.get('content')[:500] + '...') if len(self.context.get('content')) > 0 else ''
return {
'name': self.node.properties.get('stepName'),
"index": index,
'run_time': self.context.get('run_time'),
'type': self.node.type,
# 'content': self.context.get('content'), # 不保存content内容，因为content内容可能会很大
'content': content,
'status': self.status,
'err_message': self.err_message,
'document_list': self.context.get('document_list')

1
apps/application/serializers/chat_message_serializers.py
View File

@ -154,6 +154,7 @@ def get_post_handler(chat_info: ChatInfo):
details=manage.get_details(),
message_tokens=manage.context['message_tokens'],
answer_tokens=manage.context['answer_tokens'],
answer_text_list=[answer_text],
run_time=manage.context['run_time'],
index=len(chat_info.chat_record_list) + 1)
chat_info.append_chat_record(chat_record, client_id)

16
apps/application/serializers/chat_serializers.py
View File

@ -395,7 +395,8 @@ class ChatRecordSerializerModel(serializers.ModelSerializer):
class Meta:
model = ChatRecord
fields = ['id', 'chat_id', 'vote_status', 'problem_text', 'answer_text',
'message_tokens', 'answer_tokens', 'const', 'improve_paragraph_id_list', 'run_time', 'index','answer_text_list',
'message_tokens', 'answer_tokens', 'const', 'improve_paragraph_id_list', 'run_time', 'index',
'answer_text_list',
'create_time', 'update_time']
@ -457,6 +458,7 @@ class ChatRecordSerializer(serializers.Serializer):
def reset_chat_record(chat_record):
dataset_list = []
paragraph_list = []
if 'search_step' in chat_record.details and chat_record.details.get('search_step').get(
'paragraph_list') is not None:
paragraph_list = chat_record.details.get('search_step').get(
@ -468,6 +470,14 @@ class ChatRecordSerializer(serializers.Serializer):
row in
paragraph_list],
{}).items()]
if len(chat_record.improve_paragraph_id_list) > 0:
paragraph_model_list = QuerySet(Paragraph).filter(id__in=chat_record.improve_paragraph_id_list)
if len(paragraph_model_list) < len(chat_record.improve_paragraph_id_list):
paragraph_model_id_list = [str(p.id) for p in paragraph_model_list]
chat_record.improve_paragraph_id_list = list(
filter(lambda p_id: paragraph_model_id_list.__contains__(p_id),
chat_record.improve_paragraph_id_list))
chat_record.save()
return {
**ChatRecordSerializerModel(chat_record).data,
@ -608,13 +618,11 @@ class ChatRecordSerializer(serializers.Serializer):
title=instance.get("title") if 'title' in instance else '')
problem_text = instance.get('problem_text') if instance.get(
'problem_text') is not None else chat_record.problem_text
problem = Problem(id=uuid.uuid1(), content=problem_text, dataset_id=dataset_id)
problem, _ = Problem.objects.get_or_create(content=problem_text, dataset_id=dataset_id)
problem_paragraph_mapping = ProblemParagraphMapping(id=uuid.uuid1(), dataset_id=dataset_id,
document_id=document_id,
problem_id=problem.id,
paragraph_id=paragraph.id)
# 插入问题
problem.save()
# 插入段落
paragraph.save()
# 插入关联问题

3
apps/common/db/sql_execute.py
View File

@ -36,8 +36,9 @@ def update_execute(sql: str, params):
"""
with connection.cursor() as cursor:
cursor.execute(sql, params)
affected_rows = cursor.rowcount
cursor.close()
return None
return affected_rows
def select_list(sql: str, params: List):

25
apps/common/event/listener_manage.py
View File

@ -10,11 +10,12 @@ import datetime
import logging
import os
import threading
import time
import traceback
from typing import List
import django.db.models
from django.db import models
from django.db import models, transaction
from django.db.models import QuerySet
from django.db.models.functions import Substr, Reverse
from langchain_core.embeddings import Embeddings
@ -168,6 +169,7 @@ class ListenerManagement:
@staticmethod
def get_aggregation_document_status(document_id):
def aggregation_document_status():
pass
sql = get_file_content(
os.path.join(PROJECT_DIR, "apps", "dataset", 'sql', 'update_document_status_meta.sql'))
native_update({'document_custom_sql': QuerySet(Document).filter(id=document_id)}, sql, with_table_name=True)
@ -179,7 +181,8 @@ class ListenerManagement:
def aggregation_document_status():
sql = get_file_content(
os.path.join(PROJECT_DIR, "apps", "dataset", 'sql', 'update_document_status_meta.sql'))
native_update({'document_custom_sql': QuerySet(Document).filter(dataset_id=dataset_id)}, sql)
native_update({'document_custom_sql': QuerySet(Document).filter(dataset_id=dataset_id)}, sql,
with_table_name=True)
return aggregation_document_status
@ -188,7 +191,7 @@ class ListenerManagement:
def aggregation_document_status():
sql = get_file_content(
os.path.join(PROJECT_DIR, "apps", "dataset", 'sql', 'update_document_status_meta.sql'))
native_update({'document_custom_sql': queryset}, sql)
native_update({'document_custom_sql': queryset}, sql, with_table_name=True)
return aggregation_document_status
@ -247,19 +250,23 @@ class ListenerManagement:
"""
if not try_lock('embedding' + str(document_id)):
return
max_kb.info(f"开始--->向量化文档:{document_id}")
# 批量修改状态为PADDING
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING, State.STARTED)
try:
# 删除文档向量数据
VectorStore.get_embedding_vector().delete_by_document_id(document_id)
def is_the_task_interrupted():
document = QuerySet(Document).filter(id=document_id).first()
if document is None or Status(document.status)[TaskType.EMBEDDING] == State.REVOKE:
return True
return False
if is_the_task_interrupted():
return
max_kb.info(f"开始--->向量化文档:{document_id}")
# 批量修改状态为PADDING
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING,
State.STARTED)
# 删除文档向量数据
VectorStore.get_embedding_vector().delete_by_document_id(document_id)
# 根据段落进行向量化处理
page(QuerySet(Paragraph).filter(document_id=document_id).values('id'), 5,
ListenerManagement.get_embedding_paragraph_apply(embedding_model, is_the_task_interrupted,

2
apps/common/handle/impl/doc_split_handle.py
View File

@ -198,4 +198,4 @@ class DocSplitHandle(BaseSplitHandle):
return self.to_md(doc, image_list, get_image_id_func())
except BaseException as e:
traceback.print_exception(e)
return ''
return f'{e}'

2
apps/common/handle/impl/html_split_handle.py
View File

@ -70,4 +70,4 @@ class HTMLSplitHandle(BaseSplitHandle):
return html2text(content)
except BaseException as e:
traceback.print_exception(e)
return ''
return f'{e}'

2
apps/common/handle/impl/pdf_split_handle.py
View File

@ -321,4 +321,4 @@ class PdfSplitHandle(BaseSplitHandle):
return self.handle_pdf_content(file, pdf_document)
except BaseException as e:
traceback.print_exception(e)
return ''
return f'{e}'

2
apps/common/handle/impl/text_split_handle.py
View File

@ -57,4 +57,4 @@ class TextSplitHandle(BaseSplitHandle):
return buffer.decode(detect(buffer)['encoding'])
except BaseException as e:
traceback.print_exception(e)
return ''
return f'{e}'

3
apps/common/util/page_utils.py
View File

@ -18,10 +18,11 @@ def page(query_set, page_size, handler, is_the_task_interrupted=lambda: False):
@param is_the_task_interrupted: 任务是否被中断
@return:
"""
query = query_set.order_by("id")
count = query_set.count()
for i in range(0, ceil(count / page_size)):
if is_the_task_interrupted():
return
offset = i * page_size
paragraph_list = query_set[offset: offset + page_size]
paragraph_list = query.all()[offset: offset + page_size]
handler(paragraph_list)

6
apps/dataset/migrations/0011_document_status_meta_paragraph_status_meta_and_more.py
View File

@ -7,6 +7,11 @@ import dataset
from common.event import ListenerManagement
from dataset.models import State, TaskType
sql = """
UPDATE "document"
SET status ="replace"(status, '1', '3')
"""
def updateDocumentStatus(apps, schema_editor):
ParagraphModel = apps.get_model('dataset', 'Paragraph')
@ -43,5 +48,6 @@ class Migration(migrations.Migration):
name='status',
field=models.CharField(default=dataset.models.data_set.Status.__str__, max_length=20, verbose_name='状态'),
),
migrations.RunSQL(sql),
migrations.RunPython(updateDocumentStatus)
]

14
apps/dataset/serializers/document_serializers.py
View File

@ -297,6 +297,9 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
ListenerManagement.update_status(QuerySet(Document).filter(id__in=document_id_list),
TaskType.EMBEDDING,
State.PENDING)
ListenerManagement.update_status(QuerySet(Paragraph).filter(document_id__in=document_id_list),
TaskType.EMBEDDING,
State.PENDING)
embedding_by_document_list.delay(document_id_list, model_id)
else:
update_embedding_dataset_id(pid_list, target_dataset_id)
@ -613,7 +616,8 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
document_id = self.data.get("document_id")
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING,
State.PENDING)
ListenerManagement.update_status(QuerySet(Paragraph).filter(document_id=document_id), TaskType.EMBEDDING,
ListenerManagement.update_status(QuerySet(Paragraph).filter(document_id=document_id),
TaskType.EMBEDDING,
State.PENDING)
ListenerManagement.get_aggregation_document_status(document_id)()
embedding_model_id = get_embedding_model_id_by_dataset_id(dataset_id=self.data.get('dataset_id'))
@ -708,8 +712,8 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
@staticmethod
def post_embedding(result, document_id, dataset_id):
model_id = get_embedding_model_id_by_dataset_id(dataset_id)
embedding_by_document.delay(document_id, model_id)
DocumentSerializers.Operate(
data={'dataset_id': dataset_id, 'document_id': document_id}).refresh()
return result
@staticmethod
@ -907,8 +911,8 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
@staticmethod
def post_embedding(document_list, dataset_id):
for document_dict in document_list:
model_id = get_embedding_model_id_by_dataset_id(dataset_id)
embedding_by_document.delay(document_dict.get('id'), model_id)
DocumentSerializers.Operate(
data={'dataset_id': dataset_id, 'document_id': document_dict.get('id')}).refresh()
return document_list
@post(post_function=post_embedding)

12
apps/dataset/serializers/paragraph_serializers.py
View File

@ -540,8 +540,16 @@ class ParagraphSerializers(ApiMixin, serializers.Serializer):
if with_valid:
self.is_valid(raise_exception=True)
paragraph_id = self.data.get('paragraph_id')
QuerySet(Paragraph).filter(id=paragraph_id).delete()
QuerySet(ProblemParagraphMapping).filter(paragraph_id=paragraph_id).delete()
Paragraph.objects.filter(id=paragraph_id).delete()
problem_id = ProblemParagraphMapping.objects.filter(paragraph_id=paragraph_id).values_list('problem_id',
flat=True).first()
if problem_id is not None:
if ProblemParagraphMapping.objects.filter(problem_id=problem_id).count() == 1:
Problem.objects.filter(id=problem_id).delete()
ProblemParagraphMapping.objects.filter(paragraph_id=paragraph_id).delete()
update_document_char_length(self.data.get('document_id'))
delete_embedding_by_paragraph(paragraph_id)

25
apps/dataset/task/generate.py
View File

@ -51,21 +51,28 @@ def get_generate_problem(llm_model, prompt, post_apply=lambda: None, is_the_task
return generate_problem
def get_is_the_task_interrupted(document_id):
def is_the_task_interrupted():
document = QuerySet(Document).filter(id=document_id).first()
if document is None or Status(document.status)[TaskType.GENERATE_PROBLEM] == State.REVOKE:
return True
return False
return is_the_task_interrupted
@celery_app.task(base=QueueOnce, once={'keys': ['document_id']},
name='celery:generate_related_by_document')
def generate_related_by_document_id(document_id, model_id, prompt):
try:
is_the_task_interrupted = get_is_the_task_interrupted(document_id)
if is_the_task_interrupted():
return
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id),
TaskType.GENERATE_PROBLEM,
State.STARTED)
llm_model = get_llm_model(model_id)
def is_the_task_interrupted():
document = QuerySet(Document).filter(id=document_id).first()
if document is None or Status(document.status)[TaskType.GENERATE_PROBLEM] == State.REVOKE:
return True
return False
# 生成问题函数
generate_problem = get_generate_problem(llm_model, prompt,
ListenerManagement.get_aggregation_document_status(
@ -82,6 +89,12 @@ def generate_related_by_document_id(document_id, model_id, prompt):
name='celery:generate_related_by_paragraph_list')
def generate_related_by_paragraph_id_list(document_id, paragraph_id_list, model_id, prompt):
try:
is_the_task_interrupted = get_is_the_task_interrupted(document_id)
if is_the_task_interrupted():
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id),
TaskType.GENERATE_PROBLEM,
State.REVOKED)
return
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id),
TaskType.GENERATE_PROBLEM,
State.STARTED)

1
apps/embedding/task/embedding.py
View File

@ -102,6 +102,7 @@ def embedding_by_dataset(dataset_id, model_id):
max_kb.info(f"数据集文档:{[d.name for d in document_list]}")
for document in document_list:
try:
print(document.id, model_id)
embedding_by_document.delay(document.id, model_id)
except Exception as e:
pass

4
apps/smartdoc/settings/lib.py
View File

@ -32,9 +32,11 @@ CELERY_WORKER_REDIRECT_STDOUTS = True
CELERY_WORKER_REDIRECT_STDOUTS_LEVEL = "INFO"
CELERY_TASK_SOFT_TIME_LIMIT = 3600
CELERY_WORKER_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS = True
CELERY_ACKS_LATE = True
celery_once_path = os.path.join(celery_data_dir, "celery_once")
CELERY_ONCE = {
'backend': 'celery_once.backends.File',
'settings': {'location': os.path.join(celery_data_dir, "celery_once")}
'settings': {'location': celery_once_path}
}
CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
CELERY_LOG_DIR = os.path.join(PROJECT_DIR, 'logs', 'celery')

16
ui/src/components/ai-chat/ExecutionDetailDialog.vue
View File

@ -63,11 +63,10 @@
<span class="color-secondary">{{ f.label }}:</span> {{ f.value }}
</div>
<div v-if="item.document_list?.length > 0">
<p class="mb-8 color-secondary">上传的文档:</p>
<p class="mb-8 color-secondary">文档:</p>
<el-space wrap>
<template v-for="(f, i) in item.document_list" :key="i">
{{ f.name }}
<el-card
shadow="never"
style="--el-card-padding: 8px"
@ -84,7 +83,7 @@
</el-space>
</div>
<div v-if="item.image_list?.length > 0">
<p class="mb-8 color-secondary">上传的图片:</p>
<p class="mb-8 color-secondary">图片:</p>
<el-space wrap>
<template v-for="(f, i) in item.image_list" :key="i">
@ -219,7 +218,16 @@
<!-- 文档内容提取 -->
<template v-if="item.type === WorkflowType.DocumentExtractNode">
<div class="card-never border-r-4">
<h5 class="p-8-12">参数输出</h5>
<h5 class="p-8-12">
参数输出
<el-tooltip
effect="dark"
content="每个文档仅支持预览500字"
placement="right"
>
<AppIcon iconName="app-warning" class="app-warning-icon"></AppIcon>
</el-tooltip>
</h5>
<div class="p-8-12 border-t-dashed lighter">
<el-scrollbar height="150">
<MdPreview

20
ui/src/components/ai-chat/component/chat-input-operate/index.vue
View File

@ -84,14 +84,15 @@
:on-change="(file: any, fileList: any) => uploadFile(file, fileList)"
>
<el-tooltip effect="dark" placement="top" popper-class="upload-tooltip-width">
<template #content
>上传文件：最多{{
props.applicationDetails.file_upload_setting.maxFiles
}}个，每个文件限制
{{ props.applicationDetails.file_upload_setting.fileLimit }}MB<br />文件类型：{{
getAcceptList().replace(/\./g, '').replace(/,/g, '、').toUpperCase()
}}</template
>
<template #content>
<div class="break-all pre-wrap">上传文件：最多{{
props.applicationDetails.file_upload_setting.maxFiles
}}个，每个文件限制
{{ props.applicationDetails.file_upload_setting.fileLimit }}MB<br />文件类型：{{
getAcceptList().replace(/\./g, '').replace(/,/g, '、').toUpperCase()
}}
</div>
</template>
<el-button text>
<el-icon><Paperclip /></el-icon>
</el-button>
@ -216,6 +217,9 @@ const getAcceptList = () => {
accepts = [...accepts, ...videoExtensions]
}
// console.log(accepts)
if (accepts.length === 0) {
return '.请在文件上传配置中选择文件类型'
}
return accepts.map((ext: any) => '.' + ext).join(',')
}

4
ui/src/components/ai-chat/component/user-form/index.vue
View File

@ -20,7 +20,7 @@
:key="dynamicsFormRefresh"
v-model="form_data_context"
:model="form_data_context"
label-position="left"
label-position="top"
require-asterisk-position="right"
:render_data="inputFieldList"
ref="dynamicsFormRef"
@ -29,7 +29,7 @@
v-if="type === 'debug-ai-chat'"
v-model="api_form_data_context"
:model="api_form_data_context"
label-position="left"
label-position="top"
require-asterisk-position="right"
:render_data="apiInputFieldList"
ref="dynamicsFormRef2"

20
ui/src/locales/lang/zh_CN/index.ts
View File

@ -40,7 +40,7 @@ export default {
validateUrlPlaceholder: '请输入验证地址',
redirectUrl: '回调地址',
redirectUrlPlaceholder: '请输入回调地址',
enableAuthentication: '启用CAS认证',
enableAuthentication: '启用 CAS 认证',
saveSuccess: '保存成功',
save: '保存'
},
@ -49,18 +49,18 @@ export default {
authEndpoint: '授权端地址',
authEndpointPlaceholder: '请输入授权端地址',
tokenEndpoint: 'Token端地址',
tokenEndpointPlaceholder: '请输入Token端地址',
tokenEndpointPlaceholder: '请输入 Token 端地址',
userInfoEndpoint: '用户信息端地址',
userInfoEndpointPlaceholder: '请输入用户信息端地址',
clientId: '客户端ID',
clientIdPlaceholder: '请输入客户端ID',
clientId: '客户端 ID',
clientIdPlaceholder: '请输入客户端 ID',
clientSecret: '客户端密钥',
clientSecretPlaceholder: '请输入客户端密钥',
logoutEndpoint: '注销端地址',
logoutEndpointPlaceholder: '请输入注销端地址',
redirectUrl: '回调地址',
redirectUrlPlaceholder: '请输入回调地址',
enableAuthentication: '启用OIDC认证'
enableAuthentication: '启用 OIDC 认证'
},
jump_tip: '即将跳转至认证源页面进行认证',
jump: '跳转',
@ -68,21 +68,21 @@ export default {
title: 'OAUTH2 设置',
authEndpoint: '授权端地址',
authEndpointPlaceholder: '请输入授权端地址',
tokenEndpoint: 'Token端地址',
tokenEndpointPlaceholder: '请输入Token端地址',
tokenEndpoint: 'Token 端地址',
tokenEndpointPlaceholder: '请输入 Token 端地址',
userInfoEndpoint: '用户信息端地址',
userInfoEndpointPlaceholder: '请输入用户信息端地址',
scope: '连接范围',
scopePlaceholder: '请输入连接范围',
clientId: '客户端ID',
clientIdPlaceholder: '请输入客户端ID',
clientId: '客户端 ID',
clientIdPlaceholder: '请输入客户端 ID',
clientSecret: '客户端密钥',
clientSecretPlaceholder: '请输入客户端密钥',
redirectUrl: '回调地址',
redirectUrlPlaceholder: '请输入回调地址',
filedMapping: '字段映射',
filedMappingPlaceholder: '请输入字段映射',
enableAuthentication: '启用OAUTH2认证',
enableAuthentication: '启用 OAUTH2 认证',
save: '保存',
saveSuccess: '保存成功'
}

2
ui/src/views/application/ApplicationSetting.vue
View File

@ -538,7 +538,7 @@
</h4>
</div>
<div class="scrollbar-height">
<AiChat :applicationDetails="applicationForm"></AiChat>
<AiChat :applicationDetails="applicationForm" :type="'debug-ai-chat'"></AiChat>
</div>
</div>
</el-col>

115
ui/src/views/document/component/Status.vue
View File

@ -1,51 +1,13 @@
<template>
<el-popover placement="top" :width="450" trigger="hover">
<template #default>
<el-row :gutter="3" v-for="status in statusTable" :key="status.type">
<el-col :span="4">{{ taskTypeMap[status.type] }} </el-col>
<el-col :span="4">
<el-text v-if="status.state === State.SUCCESS || status.state === State.REVOKED">
<el-icon class="success"><SuccessFilled /></el-icon>
{{ stateMap[status.state](status.type) }}
</el-text>
<el-text v-else-if="status.state === State.FAILURE">
<el-icon class="danger"><CircleCloseFilled /></el-icon>
{{ stateMap[status.state](status.type) }}
</el-text>
<el-text v-else-if="status.state === State.STARTED">
<el-icon class="is-loading primary"><Loading /></el-icon>
{{ stateMap[status.state](status.type) }}
</el-text>
<el-text v-else-if="status.state === State.PENDING">
<el-icon class="is-loading primary"><Loading /></el-icon>
{{ stateMap[status.state](status.type) }}
</el-text>
<el-text v-else-if="aggStatus?.value === State.REVOKE">
<el-icon class="is-loading primary"><Loading /></el-icon>
{{ stateMap[aggStatus.value](aggStatus.key) }}
</el-text>
</el-col>
<el-col :span="5">
完成
{{
Object.keys(status.aggs ? status.aggs : {})
.filter((k) => k == State.SUCCESS)
.map((k) => status.aggs[k])
.reduce((x: any, y: any) => x + y, 0)
}}/{{
Object.values(status.aggs ? status.aggs : {}).reduce((x: any, y: any) => x + y, 0)
}}
</el-col>
<el-col :span="9">
{{
status.time
? status.time[
status.state == State.REVOKED ? State.REVOKED : State.PENDING
]?.substring(0, 19)
: undefined
}}
</el-col>
</el-row>
<el-popover v-model:visible="visible" placement="top" :width="450" trigger="hover">
<template #default
><StatusTable
v-if="visible"
:status="status"
:statusMeta="statusMeta"
:taskTypeMap="taskTypeMap"
:stateMap="stateMap"
></StatusTable>
</template>
<template #reference>
<el-text v-if="aggStatus?.value === State.SUCCESS || aggStatus?.value === State.REVOKED">
@ -72,11 +34,11 @@
</el-popover>
</template>
<script setup lang="ts">
import { computed } from 'vue'
import { Status, TaskType, State, type TaskTypeInterface } from '@/utils/status'
import { mergeWith } from 'lodash'
import { computed, ref } from 'vue'
import { TaskType, State } from '@/utils/status'
import StatusTable from '@/views/document/component/StatusTable.vue'
const props = defineProps<{ status: string; statusMeta: any }>()
const visible = ref<boolean>(false)
const checkList: Array<string> = [
State.REVOKE,
State.STARTED,
@ -112,56 +74,5 @@ const stateMap: any = {
[State.FAILURE]: (type: number) => '失败',
[State.SUCCESS]: (type: number) => '成功'
}
const parseAgg = (agg: { count: number; status: string }) => {
const status = new Status(agg.status)
return Object.keys(TaskType)
.map((key) => {
const value = TaskType[key as keyof TaskTypeInterface]
return { [value]: { [status.task_status[value]]: agg.count } }
})
.reduce((x, y) => ({ ...x, ...y }), {})
}
const customizer: (x: any, y: any) => any = (objValue: any, srcValue: any) => {
if (objValue == undefined && srcValue) {
return srcValue
}
if (srcValue == undefined && objValue) {
return objValue
}
// 如果是数组，我们将元素进行聚合
if (typeof objValue === 'object' && typeof srcValue === 'object') {
// 若是object类型的对象，我们进行递归
return mergeWith(objValue, srcValue, customizer)
} else {
// 否则，单纯的将值进行累加
return objValue + srcValue
}
}
const aggs = computed(() => {
return (props.statusMeta.aggs ? props.statusMeta.aggs : [])
.map((agg: any) => {
return parseAgg(agg)
})
.reduce((x: any, y: any) => {
return mergeWith(x, y, customizer)
}, {})
})
const statusTable = computed(() => {
return Object.keys(TaskType)
.map((key) => {
const value = TaskType[key as keyof TaskTypeInterface]
const parseStatus = new Status(props.status)
return {
type: value,
state: parseStatus.task_status[value],
aggs: aggs.value[value],
time: props.statusMeta.state_time[value]
}
})
.filter((item) => item.state !== State.IGNORED)
})
</script>
<style lang="scss" scoped></style>

110
ui/src/views/document/component/StatusTable.vue Normal file
View File

@ -0,0 +1,110 @@
<template>
<el-row :gutter="3" v-for="status in statusTable" :key="status.type">
<el-col :span="4">{{ taskTypeMap[status.type] }} </el-col>
<el-col :span="4">
<el-text v-if="status.state === State.SUCCESS || status.state === State.REVOKED">
<el-icon class="success"><SuccessFilled /></el-icon>
{{ stateMap[status.state](status.type) }}
</el-text>
<el-text v-else-if="status.state === State.FAILURE">
<el-icon class="danger"><CircleCloseFilled /></el-icon>
{{ stateMap[status.state](status.type) }}
</el-text>
<el-text v-else-if="status.state === State.STARTED">
<el-icon class="is-loading primary"><Loading /></el-icon>
{{ stateMap[status.state](status.type) }}
</el-text>
<el-text v-else-if="status.state === State.PENDING">
<el-icon class="is-loading primary"><Loading /></el-icon>
{{ stateMap[status.state](status.type) }}
</el-text>
<el-text v-else-if="status.state === State.REVOKE">
<el-icon class="is-loading primary"><Loading /></el-icon>
{{ stateMap[status.state](status.type) }}
</el-text>
</el-col>
<el-col :span="7">
<span
:style="{ color: [State.FAILURE, State.REVOKED].includes(status.state) ? '#F54A45' : '' }"
>
完成
{{
Object.keys(status.aggs ? status.aggs : {})
.filter((k) => k == State.SUCCESS)
.map((k) => status.aggs[k])
.reduce((x: any, y: any) => x + y, 0)
}}/{{
Object.values(status.aggs ? status.aggs : {}).reduce((x: any, y: any) => x + y, 0)
}}</span
>
</el-col>
<el-col :span="9">
{{
status.time
? status.time[status.state == State.REVOKED ? State.REVOKED : State.PENDING]?.substring(
0,
19
)
: undefined
}}
</el-col>
</el-row>
</template>
<script setup lang="ts">
import { computed } from 'vue'
import { Status, TaskType, State, type TaskTypeInterface } from '@/utils/status'
import { mergeWith } from 'lodash'
const props = defineProps<{ status: string; statusMeta: any; stateMap: any; taskTypeMap: any }>()
const parseAgg = (agg: { count: number; status: string }) => {
const status = new Status(agg.status)
return Object.keys(TaskType)
.map((key) => {
const value = TaskType[key as keyof TaskTypeInterface]
return { [value]: { [status.task_status[value]]: agg.count } }
})
.reduce((x, y) => ({ ...x, ...y }), {})
}
const customizer: (x: any, y: any) => any = (objValue: any, srcValue: any) => {
if (objValue == undefined && srcValue) {
return srcValue
}
if (srcValue == undefined && objValue) {
return objValue
}
// 如果是数组，我们将元素进行聚合
if (typeof objValue === 'object' && typeof srcValue === 'object') {
// 若是object类型的对象，我们进行递归
return mergeWith(objValue, srcValue, customizer)
} else {
// 否则，单纯的将值进行累加
return objValue + srcValue
}
}
const aggs = computed(() => {
return (props.statusMeta.aggs ? props.statusMeta.aggs : [])
.map((agg: any) => {
return parseAgg(agg)
})
.reduce((x: any, y: any) => {
return mergeWith(x, y, customizer)
}, {})
})
const statusTable = computed(() => {
return Object.keys(TaskType)
.map((key) => {
const value = TaskType[key as keyof TaskTypeInterface]
const parseStatus = new Status(props.status)
return {
type: value,
state: parseStatus.task_status[value],
aggs: aggs.value[value],
time: props.statusMeta.state_time[value]
}
})
.filter((item) => item.state !== State.IGNORED)
})
</script>
<style lang="scss"></style>

45
ui/src/views/document/index.vue
View File

@ -235,7 +235,25 @@
<template #default="{ row }">
<div v-if="datasetDetail.type === '0'">
<span class="mr-4">
<el-tooltip effect="dark" content="向量化" placement="top">
<el-tooltip
effect="dark"
v-if="
([State.STARTED, State.PENDING] as Array<string>).includes(
getTaskState(row.status, TaskType.EMBEDDING)
)
"
content="取消向量化"
placement="top"
>
<el-button
type="primary"
text
@click.stop="cancelTask(row, TaskType.EMBEDDING)"
>
<AppIcon iconName="app-close" style="font-size: 16px"></AppIcon>
</el-button>
</el-tooltip>
<el-tooltip v-else effect="dark" content="向量化" placement="top">
<el-button type="primary" text @click.stop="refreshDocument(row)">
<AppIcon iconName="app-document-refresh" style="font-size: 16px"></AppIcon>
</el-button>
@ -255,9 +273,20 @@
</el-button>
<template #dropdown>
<el-dropdown-menu>
<el-dropdown-item @click="openGenerateDialog(row)">
<el-dropdown-item
v-if="
([State.STARTED, State.PENDING] as Array<string>).includes(
getTaskState(row.status, TaskType.GENERATE_PROBLEM)
)
"
@click="cancelTask(row, TaskType.GENERATE_PROBLEM)"
>
<el-icon><Connection /></el-icon>
生成关联问题
取消生成问题
</el-dropdown-item>
<el-dropdown-item v-else @click="openGenerateDialog(row)">
<el-icon><Connection /></el-icon>
生成问题
</el-dropdown-item>
<el-dropdown-item @click="openDatasetDialog(row)">
<AppIcon iconName="app-migrate"></AppIcon>
@ -286,7 +315,11 @@
<span class="mr-4">
<el-tooltip
effect="dark"
v-if="getTaskState(row.status, TaskType.EMBEDDING) == State.STARTED"
v-if="
([State.STARTED, State.PENDING] as Array<string>).includes(
getTaskState(row.status, TaskType.EMBEDDING)
)
"
content="取消向量化"
placement="top"
>
@ -318,7 +351,9 @@
>
<el-dropdown-item
v-if="
getTaskState(row.status, TaskType.GENERATE_PROBLEM) == State.STARTED
([State.STARTED, State.PENDING] as Array<string>).includes(
getTaskState(row.status, TaskType.GENERATE_PROBLEM)
)
"
@click="cancelTask(row, TaskType.GENERATE_PROBLEM)"
>

2
ui/src/workflow/common/data.ts
View File

@ -203,7 +203,7 @@ export const documentExtractNode = {
config: {
fields: [
{
label: '文件内容',
label: '文档内容',
value: 'content'
}
]

29
ui/src/workflow/nodes/image-understand/index.vue
View File

@ -56,7 +56,7 @@
></span>
<span>{{ item.name }}</span>
<el-tag v-if="item.permission_type === 'PUBLIC'" type="info" class="info-tag ml-8"
>公用
>公用
</el-tag>
</div>
<el-icon class="check-icon" v-if="item.id === form_data.model_id">
@ -113,7 +113,7 @@
</div>
<el-tooltip effect="dark" placement="right" popper-class="max-w-200">
<template #content
>通过调整提示词内容，可以引导大模型聊天方向，该提示词会被固定在上下文的开头，可以使用变量。
>通过调整提示词内容，可以引导大模型聊天方向，该提示词会被固定在上下文的开头，可以使用变量。
</template>
<AppIcon iconName="app-warning" class="app-warning-icon"></AppIcon>
</el-tooltip>
@ -131,9 +131,9 @@
<template #label>
<div class="flex-between">
<div>历史聊天记录</div>
<el-select v-model="form_data.dialogue_type" type="small" style="width: 100px;">
<el-option label="节点" value="NODE"/>
<el-option label="工作流" value="WORKFLOW"/>
<el-select v-model="form_data.dialogue_type" type="small" style="width: 100px">
<el-option label="节点" value="NODE" />
<el-option label="工作流" value="WORKFLOW" />
</el-select>
</div>
</template>
@ -143,9 +143,13 @@
:value-on-clear="0"
controls-position="right"
class="w-full"
:step="1"
:step-strictly="true"
/>
</el-form-item>
<el-form-item label="选择图片" :rules="{
<el-form-item
label="选择图片"
:rules="{
type: 'array',
required: true,
message: '请选择图片',
@ -232,7 +236,7 @@ const form = {
is_result: true,
temperature: null,
max_tokens: null,
image_list: ["start-node", "image"]
image_list: ['start-node', 'image']
}
const form_data = computed({
@ -249,7 +253,6 @@ const form_data = computed({
}
})
function getModel() {
if (id) {
applicationApi.getApplicationImageModel(id).then((res: any) => {
@ -268,9 +271,7 @@ function getProvider() {
})
}
const model_change = (model_id?: string) => {
}
const model_change = (model_id?: string) => {}
function submitSystemDialog(val: string) {
set(props.nodeModel.properties.node_data, 'system', val)
@ -286,10 +287,6 @@ onMounted(() => {
set(props.nodeModel, 'validate', validate)
})
</script>
<style scoped lang="scss">
</style>
<style scoped lang="scss"></style>