From c67f4633a1a9a1f15e1fb807b63bf56c13712e7c Mon Sep 17 00:00:00 2001 From: wanghe Date: Sun, 7 Apr 2024 23:18:36 +0800 Subject: [PATCH 1/2] =?UTF-8?q?chore=EF=BC=9A=E5=A2=9E=E5=8A=A0=20Star=20H?= =?UTF-8?q?istory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 216c76497..87fcf55e7 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,10 @@ docker run -d --name=maxkb -p 8080:8080 -v ~/.maxkb:/var/lib/postgresql/data 1pa - 向量数据库：[PostgreSQL / pgvector](https://www.postgresql.org/) - 大模型：Azure OpenAI、百度千帆大模型、[Ollama](https://github.com/ollama/ollama) +## Star History + +[![Star History Chart](https://api.star-history.com/svg?repos=1Panel-dev/MaxKB&type=Date)](https://star-history.com/#1Panel-dev/MaxKB&Date) + ## License Copyright (c) 2014-2024 飞致云 FIT2CLOUD, All rights reserved. From 633c0059054aca0696268849b73a8cfd989ef0af Mon Sep 17 00:00:00 2001 From: shaohuzhang1 <80892890+shaohuzhang1@users.noreply.github.com> Date: Tue, 9 Apr 2024 11:33:28 +0800 Subject: [PATCH 2/2] Pr@main@problem manage (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 问题模块 --- .../serializers/paragraph_serializers.py | 2 +- .../serializers/problem_serializers.py | 22 +- apps/dataset/swagger_api/problem_api.py | 19 + apps/dataset/urls.py | 1 + apps/dataset/views/problem.py | 20 +- ui/src/api/paragraph.ts | 47 ++- ui/src/api/problem.ts | 107 ++++++ .../ai-chat/ParagraphSourceDialog.vue | 9 - ui/src/components/app-table/index.vue | 20 +- ui/src/components/icons/index.ts | 46 +++ ui/src/components/read-write/index.vue | 23 +- ui/src/router/modules/dataset.ts | 12 + ui/src/stores/index.ts | 4 +- ui/src/stores/modules/document.ts | 14 +- ui/src/stores/modules/problem.ts | 79 ++++ ui/src/styles/app.scss | 33 ++ ui/src/views/dataset/CreateDataset.vue | 20 +- .../views/dataset/component/SyncWebDialog.vue | 4 +- ui/src/views/dataset/step/StepFirst.vue | 4 +- ui/src/views/dataset/step/StepSecond.vue | 4 +- ui/src/views/document/index.vue | 10 +- .../paragraph/component/ParagraphDialog.vue | 21 +- .../paragraph/component/ProblemComponent.vue | 115 +++--- .../problem/component/CreateProblemDialog.vue | 90 +++++ .../problem/component/DetailProblemDrawer.vue | 197 ++++++++++ .../problem/component/RelateProblemDialog.vue | 261 ++++++++++++++ ui/src/views/problem/index.vue | 340 ++++++++++++++++++ 27 files changed, 1405 insertions(+), 119 deletions(-) create mode 100644 ui/src/api/problem.ts create mode 100644 ui/src/stores/modules/problem.ts create mode 100644 ui/src/views/problem/component/CreateProblemDialog.vue create mode 100644 ui/src/views/problem/component/DetailProblemDrawer.vue create mode 100644 ui/src/views/problem/component/RelateProblemDialog.vue create mode 100644 ui/src/views/problem/index.vue diff --git a/apps/dataset/serializers/paragraph_serializers.py b/apps/dataset/serializers/paragraph_serializers.py index 023f4cecd..0b69e8e3f 100644 --- a/apps/dataset/serializers/paragraph_serializers.py +++ b/apps/dataset/serializers/paragraph_serializers.py @@ -218,7 +218,7 @@ class ParagraphSerializers(ApiMixin, serializers.Serializer): def association(self, with_valid=True, with_embedding=True): if with_valid: self.is_valid(raise_exception=True) - problem = QuerySet(Problem).filter(id=self.data.get("problem_id")) + problem = QuerySet(Problem).filter(id=self.data.get("problem_id")).first() problem_paragraph_mapping = ProblemParagraphMapping(id=uuid.uuid1(), document_id=self.data.get('document_id'), paragraph_id=self.data.get('paragraph_id'), diff --git a/apps/dataset/serializers/problem_serializers.py b/apps/dataset/serializers/problem_serializers.py index a1bf6df91..2587e9712 100644 --- a/apps/dataset/serializers/problem_serializers.py +++ b/apps/dataset/serializers/problem_serializers.py @@ -8,7 +8,7 @@ """ import os import uuid -from typing import Dict +from typing import Dict, List from django.db import transaction from django.db.models import QuerySet @@ -83,6 +83,7 @@ class ProblemSerializers(ApiMixin, serializers.Serializer): **{'dataset_id': self.data.get('dataset_id')}) if 'content' in self.data: query_set = query_set.filter(**{'content__contains': self.data.get('content')}) + query_set = query_set.order_by("-create_time") return query_set def list(self): @@ -95,6 +96,22 @@ class ProblemSerializers(ApiMixin, serializers.Serializer): return native_page_search(current_page, page_size, query_set, select_string=get_file_content( os.path.join(PROJECT_DIR, "apps", "dataset", 'sql', 'list_problem.sql'))) + class BatchOperate(serializers.Serializer): + dataset_id = serializers.UUIDField(required=True, error_messages=ErrMessage.uuid("知识库id")) + + def delete(self, problem_id_list: List, with_valid=True): + if with_valid: + self.is_valid(raise_exception=True) + dataset_id = self.data.get('dataset_id') + problem_paragraph_mapping_list = QuerySet(ProblemParagraphMapping).filter( + dataset_id=dataset_id, + problem_id__in=problem_id_list) + source_ids = [row.id for row in problem_paragraph_mapping_list] + problem_paragraph_mapping_list.delete() + QuerySet(Problem).filter(id__in=problem_id_list).delete() + ListenerManagement.delete_embedding_by_source_ids_signal.send(source_ids) + return True + class Operate(serializers.Serializer): dataset_id = serializers.UUIDField(required=True, error_messages=ErrMessage.uuid("知识库id")) @@ -105,6 +122,8 @@ class ProblemSerializers(ApiMixin, serializers.Serializer): self.is_valid(raise_exception=True) problem_paragraph_mapping = QuerySet(ProblemParagraphMapping).filter(dataset_id=self.data.get("dataset_id"), problem_id=self.data.get("problem_id")) + if problem_paragraph_mapping is None or len(problem_paragraph_mapping)==0: + return [] return native_search( QuerySet(Paragraph).filter(id__in=[row.paragraph_id for row in problem_paragraph_mapping]), select_string=get_file_content( @@ -123,6 +142,7 @@ class ProblemSerializers(ApiMixin, serializers.Serializer): dataset_id=self.data.get('dataset_id'), problem_id=self.data.get('problem_id')) source_ids = [row.id for row in problem_paragraph_mapping_list] + problem_paragraph_mapping_list.delete() QuerySet(Problem).filter(id=self.data.get('problem_id')).delete() ListenerManagement.delete_embedding_by_source_ids_signal.send(source_ids) return True diff --git a/apps/dataset/swagger_api/problem_api.py b/apps/dataset/swagger_api/problem_api.py index 6e57ad4b6..a7397aaaf 100644 --- a/apps/dataset/swagger_api/problem_api.py +++ b/apps/dataset/swagger_api/problem_api.py @@ -36,6 +36,25 @@ class ProblemApi(ApiMixin): } ) + class BatchOperate(ApiMixin): + @staticmethod + def get_request_params_api(): + return [openapi.Parameter(name='dataset_id', + in_=openapi.IN_PATH, + type=openapi.TYPE_STRING, + required=True, + description='知识库id'), + ] + + @staticmethod + def get_request_body_api(): + return openapi.Schema( + title="问题id列表", + description="问题id列表", + type=openapi.TYPE_ARRAY, + items=openapi.Schema(type=openapi.TYPE_STRING) + ) + class Operate(ApiMixin): @staticmethod def get_request_params_api(): diff --git a/apps/dataset/urls.py b/apps/dataset/urls.py index d153d7134..38a92c845 100644 --- a/apps/dataset/urls.py +++ b/apps/dataset/urls.py @@ -36,6 +36,7 @@ urlpatterns = [ 'dataset//document//paragraph//problem//association', views.Paragraph.Problem.Association.as_view()), path('dataset//problem', views.Problem.as_view()), + path('dataset//problem/_batch', views.Problem.OperateBatch.as_view()), path('dataset//problem//', views.Problem.Page.as_view()), path('dataset//problem/', views.Problem.Operate.as_view()), path('dataset//problem//paragraph', views.Problem.Paragraph.as_view()), diff --git a/apps/dataset/views/problem.py b/apps/dataset/views/problem.py index 36731425f..beebcc673 100644 --- a/apps/dataset/views/problem.py +++ b/apps/dataset/views/problem.py @@ -51,7 +51,7 @@ class Problem(APIView): def post(self, request: Request, dataset_id: str): return result.success( ProblemSerializers.Create( - data={'dataset_id': dataset_id, 'problem_list': request.query_params.get('problem_list')}).save()) + data={'dataset_id': dataset_id, 'problem_list': request.data}).batch()) class Paragraph(APIView): authentication_classes = [TokenAuth] @@ -70,6 +70,24 @@ class Problem(APIView): data={**query_params_to_single_dict(request.query_params), 'dataset_id': dataset_id, 'problem_id': problem_id}).list_paragraph()) + class OperateBatch(APIView): + authentication_classes = [TokenAuth] + + @action(methods=['DELETE'], detail=False) + @swagger_auto_schema(operation_summary="批量删除问题", + operation_id="批量删除问题", + request_body= + ProblemApi.BatchOperate.get_request_body_api(), + manual_parameters=ProblemApi.BatchOperate.get_request_params_api(), + responses=result.get_default_response(), + tags=["知识库/文档/段落/问题"]) + @has_permissions( + lambda r, k: Permission(group=Group.DATASET, operate=Operate.MANAGE, + dynamic_tag=k.get('dataset_id'))) + def delete(self, request: Request, dataset_id: str): + return result.success( + ProblemSerializers.BatchOperate(data={'dataset_id': dataset_id}).delete(request.data)) + class Operate(APIView): authentication_classes = [TokenAuth] diff --git a/ui/src/api/paragraph.ts b/ui/src/api/paragraph.ts index 3df63b770..fc1169a57 100644 --- a/ui/src/api/paragraph.ts +++ b/ui/src/api/paragraph.ts @@ -129,25 +129,55 @@ const postProblem: ( dataset_id: string, document_id: string, paragraph_id: string, - data: any -) => Promise> = (dataset_id, document_id, paragraph_id, data: any) => { + data: any, + loading?: Ref +) => Promise> = (dataset_id, document_id, paragraph_id, data: any, loading) => { return post( `${prefix}/${dataset_id}/document/${document_id}/paragraph/${paragraph_id}/problem`, - data + data, + {}, + loading + ) +} +/** + * + * @param dataset_id 数据集id + * @param document_id 文档id + * @param paragraph_id 段落id + * @param problem_id 问题id + * @param loading 加载器 + * @returns + */ +const associationProblem: ( + dataset_id: string, + document_id: string, + paragraph_id: string, + problem_id: string, + loading?: Ref +) => Promise> = (dataset_id, document_id, paragraph_id, problem_id, loading) => { + return put( + `${prefix}/${dataset_id}/document/${document_id}/paragraph/${paragraph_id}/problem/${problem_id}/association`, + {}, + {}, + loading ) } /** * 解除关联问题 * @param 参数 dataset_id, document_id, paragraph_id,problem_id */ -const delProblem: ( +const disassociationProblem: ( dataset_id: string, document_id: string, paragraph_id: string, - problem_id: string -) => Promise> = (dataset_id, document_id, paragraph_id, problem_id) => { + problem_id: string, + loading?: Ref +) => Promise> = (dataset_id, document_id, paragraph_id, problem_id, loading) => { return put( - `${prefix}/${dataset_id}/document/${document_id}/paragraph/${paragraph_id}/problem/${problem_id}/un_association` + `${prefix}/${dataset_id}/document/${document_id}/paragraph/${paragraph_id}/problem/${problem_id}/un_association`, + {}, + {}, + loading ) } @@ -158,5 +188,6 @@ export default { postParagraph, getProblem, postProblem, - delProblem + disassociationProblem, + associationProblem } diff --git a/ui/src/api/problem.ts b/ui/src/api/problem.ts new file mode 100644 index 000000000..7d8d16226 --- /dev/null +++ b/ui/src/api/problem.ts @@ -0,0 +1,107 @@ +import { Result } from '@/request/Result' +import { get, post, del, put } from '@/request/index' +import type { Ref } from 'vue' +import type { KeyValue } from '@/api/type/common' +import type { pageRequest } from '@/api/type/common' +const prefix = '/dataset' + +/** + * 文档分页列表 + * @param 参数 dataset_id, + * page { + "current_page": "string", + "page_size": "string", + } +* query { + "content": "string", + } + */ + +const getProblems: ( + dataset_id: string, + page: pageRequest, + param: any, + loading?: Ref +) => Promise> = (dataset_id, page, param, loading) => { + return get( + `${prefix}/${dataset_id}/problem/${page.current_page}/${page.page_size}`, + param, + loading + ) +} + +/** + * 创建问题 + * @param 参数 dataset_id + * data: array[string] + */ +const postProblems: ( + dataset_id: string, + data: any, + loading?: Ref +) => Promise> = (dataset_id, data, loading) => { + return post(`${prefix}/${dataset_id}/problem`, data, undefined, loading) +} + +/** + * 删除问题 + * @param 参数 dataset_id, problem_id, + */ +const delProblems: ( + dataset_id: string, + problem_id: string, + loading?: Ref +) => Promise> = (dataset_id, problem_id, loading) => { + return del(`${prefix}/${dataset_id}/problem/${problem_id}`, loading) +} + +/** + * 批量删除问题 + * @param 参数 dataset_id, + */ +const delMulProblem: ( + dataset_id: string, + data: any, + loading?: Ref +) => Promise> = (dataset_id, data, loading) => { + return del(`${prefix}/${dataset_id}/problem/_batch`, undefined, data, loading) +} + +/** + * 修改问题 + * @param 参数 + * dataset_id, problem_id, + * { + "content": "string", + } + */ +const putProblems: ( + dataset_id: string, + problem_id: string, + data: any, + loading?: Ref +) => Promise> = (dataset_id, problem_id, data: any, loading) => { + return put(`${prefix}/${dataset_id}/problem/${problem_id}`, data, undefined, loading) +} + +/** + * 问题详情 + * @param 参数 + * dataset_id, problem_id, + */ +const getDetailProblems: ( + dataset_id: string, + problem_id: string, + loading?: Ref +) => Promise> = (dataset_id, problem_id, loading) => { + return get(`${prefix}/${dataset_id}/problem/${problem_id}/paragraph`, undefined, loading) +} + +export default { + getProblems, + postProblems, + delProblems, + putProblems, + getDetailProblems, + delMulProblem +} diff --git a/ui/src/components/ai-chat/ParagraphSourceDialog.vue b/ui/src/components/ai-chat/ParagraphSourceDialog.vue index d43a503bb..1b7cf2d70 100644 --- a/ui/src/components/ai-chat/ParagraphSourceDialog.vue +++ b/ui/src/components/ai-chat/ParagraphSourceDialog.vue @@ -102,13 +102,4 @@ defineExpose({ open }) height: calc(100vh - 260px); } } -.paragraph-source-card { - height: 210px; - width: 100%; - .active-button { - position: absolute; - right: 16px; - top: 16px; - } -} diff --git a/ui/src/components/app-table/index.vue b/ui/src/components/app-table/index.vue index ec15932ea..29335ba0a 100644 --- a/ui/src/components/app-table/index.vue +++ b/ui/src/components/app-table/index.vue @@ -6,9 +6,11 @@ 创建 @@ -17,7 +19,7 @@
- 快速创建空白文档 + {{ quickCreatePlaceholder }}
@@ -51,6 +53,18 @@ const props = defineProps({ quickCreate: { type: Boolean, default: false + }, + quickCreateName: { + type: String, + default: '文档名称' + }, + quickCreatePlaceholder: { + type: String, + default: '快速创建空白文档' + }, + quickCreateMaxlength: { + type: Number, + default: () => 0 } }) const emit = defineEmits(['changePage', 'sizeChange', 'creatQuick']) @@ -81,7 +95,7 @@ function submitHandle() { loading.value = false }, 200) } else { - MsgError('文件名称不能为空！') + MsgError(`${props.quickCreateName}不能为空！`) } } diff --git a/ui/src/components/icons/index.ts b/ui/src/components/icons/index.ts index b093c63bc..16e2fac23 100644 --- a/ui/src/components/icons/index.ts +++ b/ui/src/components/icons/index.ts @@ -728,5 +728,51 @@ export const iconMap: any = { ) ]) } + }, + 'app-problems': { + iconReader: () => { + return h('i', [ + h( + 'svg', + { + style: { height: '100%', width: '100%' }, + viewBox: '0 0 1024 1024', + version: '1.1', + xmlns: 'http://www.w3.org/2000/svg' + }, + [ + h('path', { + d: 'M565.03091564 528.45078523a588.83471385 588.83471385 0 0 1 16.90811971-15.58251253c16.81532721-14.88656875 34.70439623-28.84521246 50.33330501-44.73261462 28.33485369-28.83195638 37.04409293-63.99368709 29.02416942-101.57465094-9.23948212-43.27444672-40.20566608-71.52976398-84.66653122-81.02111147-31.27770165-8.21876458-35.38708395-7.01909007-67.9373685-4.33473551-37.94550581 6.16407344-39.35727747 6.05802485-76.22241344 22.62811474-2.48551348 1.39188755-19.28758462 10.35962019-24.5966414 15.11855-11.44661809 5.60731841-19.40026124 17.25940562-19.40026123 30.86013539a34.46578693 34.46578693 0 0 0 34.46578694 34.46578694 34.1807814 34.1807814 0 0 0 20.83854503-7.17816293c0.35128591-0.22535322 0.69594378-0.41756626 1.06711379-0.74896807 28.77230406-25.79631593 62.90668921-36.7259472 102.56885634-31.38375021 15.43006769 2.07457524 28.54032281 8.45737387 38.05818242 20.42097876 12.23535436 15.3770434 10.79707056 32.51714437 6.85338917 49.71026962-3.05552458 13.30909618-11.26103308 24.31163586-21.66704951 33.43181333-17.02079632 14.932965-34.65799999 29.27603478-52.28194758 43.60584853-19.63224249 15.97356663-28.85846852 36.7259472-31.52293898 60.18919446a257.89025081 257.89025081 0 0 0-1.49793613 30.30338037h0.04639624c-0.03976821 19.12188371 16.21880398 32.68947331 30.90653165 33.12029565 20.02329661 0.59652323 35.11533446-13.47479709 35.32743162-32.39783973-0.00662803-1.0869979-0.19884108-2.07457524-0.28500555-3.12180494-0.00662803-5.1433559-0.0927925-10.29333983 0.01988411-15.43006769 0.29826162-13.49468121 3.10854885-26.22713825 13.66038209-36.34814915zM515.93042532 643.75209862c-19.01583514-0.76222413-32.4309799 15.15169019-33.41192923 31.12525684-1.23281469 20.1691134 15.69518913 34.65799999 30.89327557 35.10870642 20.02329661 0.59652323 35.11533446-13.47479709 35.32743161-32.39783973-0.13918876-19.99015643-13.38863262-33.06064333-32.80877795-33.83612353zM96.72703555 251.52481518h120.80258323c17.31242991 0 31.34398202-14.84017249 31.34398202-33.14017976s-14.03818015-33.14017975-31.34398202-33.14017975H96.72703555c-17.31242991 0-31.34398202 14.84017249-31.34398201 33.14017975s14.03155212 33.14017975 31.34398201 33.14017976zM94.63920422 412.78492985h120.80258324c17.31242991 0 31.34398202-14.84017249 31.34398201-33.14017974s-14.03818015-33.14017975-31.34398201-33.14017976H94.63920422c-17.31242991 0-31.35061005 14.84017249-31.35061003 33.14017976s14.03818015 33.14017975 31.35061003 33.14017974zM246.78576947 542.32989251c0-18.3066353-14.03818015-33.14017975-31.34398201-33.14017975H94.63920422c-17.31242991 0-31.35061005 14.83354446-31.35061003 33.14017975 0 18.30000725 14.03818015 33.14017975 31.35061003 33.14017976h120.80258324c17.30580187 0 31.34398202-14.84017249 31.34398201-33.14017976z', + fill: 'currentColor' + }), + h('path', { + d: 'M824.35945025 44.76986174H194.99429654a35.93058289 35.93058289 0 0 0 0 71.84790971h629.36515371c19.80457142 0 35.96372307 16.13263951 35.96372307 35.93058289v718.5652615a35.99023521 35.99023521 0 0 1-35.96372307 35.93721092H230.10963102a35.95709503 35.95709503 0 0 1-35.95709503-35.93721092v-190.42347285a35.93721092 35.93721092 0 0 0-35.96372307-35.92395486 35.92395486 35.92395486 0 0 0-35.95709503 35.92395486v190.42347285c0 59.43359837 48.40454655 107.79837669 107.87791313 107.7983767h594.24981923c59.47999461 0 107.8712851-48.36477833 107.87128509-107.7983767V152.55498237c0-59.42697034-48.39129049-107.78512063-107.87128509-107.78512063z', + fill: 'currentColor' + }) + ] + ) + ]) + } + }, + 'app-quxiaoguanlian': { + iconReader: () => { + return h('i', [ + h( + 'svg', + { + style: { height: '100%', width: '100%' }, + viewBox: '0 0 1024 1024', + version: '1.1', + xmlns: 'http://www.w3.org/2000/svg' + }, + [ + h('path', { + d: 'M544 298.688a32 32 0 0 1 32-32h320c41.216 0 74.688 33.408 74.688 74.624V640c0 41.216-33.472 74.688-74.688 74.688h-85.312a32 32 0 1 1 0-64H896a10.688 10.688 0 0 0 10.688-10.688V341.312A10.688 10.688 0 0 0 896 330.688H576a32 32 0 0 1-32-32zM53.312 341.312c0-41.216 33.472-74.624 74.688-74.624h106.688a32 32 0 1 1 0 64H128a10.688 10.688 0 0 0-10.688 10.624V640c0 5.888 4.8 10.688 10.688 10.688h320a32 32 0 1 1 0 64H128A74.688 74.688 0 0 1 53.312 640V341.312zM282.432 100.416a32 32 0 0 1 43.84 11.392l426.624 725.312a32 32 0 0 1-55.168 32.448L271.104 144.256a32 32 0 0 1 11.328-43.84zM650.688 490.688a32 32 0 0 1 32-32H768a32 32 0 1 1 0 64h-85.312a32 32 0 0 1-32-32zM224 490.688a32 32 0 0 1 32-32h85.312a32 32 0 1 1 0 64H256a32 32 0 0 1-32-32z', + fill: 'currentColor' + }) + ] + ) + ]) + } } } diff --git a/ui/src/components/read-write/index.vue b/ui/src/components/read-write/index.vue index 28758ec70..6a04e4633 100644 --- a/ui/src/components/read-write/index.vue +++ b/ui/src/components/read-write/index.vue @@ -1,20 +1,27 @@