diff --git a/client/public/imgs/files/csv.svg b/client/public/imgs/files/csv.svg new file mode 100644 index 000000000..8e8d3f27f --- /dev/null +++ b/client/public/imgs/files/csv.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/public/imgs/files/doc.svg b/client/public/imgs/files/doc.svg new file mode 100644 index 000000000..72c75476a --- /dev/null +++ b/client/public/imgs/files/doc.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/public/imgs/files/html.svg b/client/public/imgs/files/html.svg new file mode 100644 index 000000000..b1e7cb9c1 --- /dev/null +++ b/client/public/imgs/files/html.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/public/imgs/files/markdown.svg b/client/public/imgs/files/markdown.svg new file mode 100644 index 000000000..f53dbba77 --- /dev/null +++ b/client/public/imgs/files/markdown.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/public/imgs/files/pdf.svg b/client/public/imgs/files/pdf.svg new file mode 100644 index 000000000..90940bb31 --- /dev/null +++ b/client/public/imgs/files/pdf.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/public/imgs/files/txt.svg b/client/public/imgs/files/txt.svg new file mode 100644 index 000000000..7f941fc7f --- /dev/null +++ b/client/public/imgs/files/txt.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/src/api/plugins/kb.ts b/client/src/api/plugins/kb.ts index bd117556f..71315a10e 100644 --- a/client/src/api/plugins/kb.ts +++ b/client/src/api/plugins/kb.ts @@ -11,6 +11,7 @@ import { Response as SearchTestResponse } from '@/pages/api/openapi/kb/searchTest'; import { Response as KbDataItemType } from '@/pages/api/plugins/kb/data/getDataById'; +import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData'; export type KbUpdateParams = { id: string; @@ -71,8 +72,7 @@ export const postKbDataFromList = (data: PushDataProps) => /** * 更新一条数据 */ -export const putKbDataById = (data: { dataId: string; a: string; q?: string }) => - PUT('/openapi/kb/updateData', data); +export const putKbDataById = (data: UpdateDataProps) => PUT('/openapi/kb/updateData', data); /** * 删除一条知识库数据 */ diff --git a/client/src/api/request.ts b/client/src/api/request.ts index 354716757..6ce4c0594 100644 --- a/client/src/api/request.ts +++ b/client/src/api/request.ts @@ -63,6 +63,9 @@ function responseError(err: any) { ); return Promise.reject({ message: 'token过期，重新登录' }); } + if (err?.response?.data) { + return Promise.reject(err?.response?.data); + } return Promise.reject(err); } diff --git a/client/src/components/Icon/close.tsx b/client/src/components/Icon/close.tsx new file mode 100644 index 000000000..cb8e4eef8 --- /dev/null +++ b/client/src/components/Icon/close.tsx @@ -0,0 +1,22 @@ +import React from 'react'; +import { Flex, type FlexProps } from '@chakra-ui/react'; +import MyIcon from '@/components/Icon'; + +const CloseIcon = (props: FlexProps) => { + return ( + + + + ); +}; + +export default CloseIcon; diff --git a/client/src/components/Icon/delete.tsx b/client/src/components/Icon/delete.tsx new file mode 100644 index 000000000..236934c91 --- /dev/null +++ b/client/src/components/Icon/delete.tsx @@ -0,0 +1,25 @@ +import React from 'react'; +import MyIcon from '@/components/Icon'; +import { IconProps } from '@chakra-ui/react'; + +const DeleteIcon = (props: IconProps) => { + return ( + + ); +}; + +export default DeleteIcon; + +export const hoverDeleteStyles = { + '& .delete': { + display: 'block' + } +}; diff --git a/client/src/components/Icon/icons/file/csv.svg b/client/src/components/Icon/icons/file/csv.svg new file mode 100644 index 000000000..cf58fb704 --- /dev/null +++ b/client/src/components/Icon/icons/file/csv.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/src/components/Icon/icons/file/indexImport.svg b/client/src/components/Icon/icons/file/indexImport.svg new file mode 100644 index 000000000..9b4fc9db9 --- /dev/null +++ b/client/src/components/Icon/icons/file/indexImport.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/src/components/Icon/icons/file/manualImport.svg b/client/src/components/Icon/icons/file/manualImport.svg new file mode 100644 index 000000000..9200da1df --- /dev/null +++ b/client/src/components/Icon/icons/file/manualImport.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/src/components/Icon/icons/file/qaImport.svg b/client/src/components/Icon/icons/file/qaImport.svg new file mode 100644 index 000000000..c36dd685b --- /dev/null +++ b/client/src/components/Icon/icons/file/qaImport.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/src/components/Icon/icons/file/uploadFile.svg b/client/src/components/Icon/icons/file/uploadFile.svg new file mode 100644 index 000000000..92b5b8728 --- /dev/null +++ b/client/src/components/Icon/icons/file/uploadFile.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/src/components/Icon/icons/light/close.svg b/client/src/components/Icon/icons/light/close.svg new file mode 100644 index 000000000..9fc9b8537 --- /dev/null +++ b/client/src/components/Icon/icons/light/close.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/src/components/Icon/icons/light/import.svg b/client/src/components/Icon/icons/light/import.svg new file mode 100644 index 000000000..e62871c3b --- /dev/null +++ b/client/src/components/Icon/icons/light/import.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/src/components/Icon/index.tsx b/client/src/components/Icon/index.tsx index 0b4f730ff..32d1b0060 100644 --- a/client/src/components/Icon/index.tsx +++ b/client/src/components/Icon/index.tsx @@ -54,7 +54,14 @@ const map = { voice: require('./icons/voice.svg').default, html: require('./icons/file/html.svg').default, pdf: require('./icons/file/pdf.svg').default, - markdown: require('./icons/file/markdown.svg').default + markdown: require('./icons/file/markdown.svg').default, + importLight: require('./icons/light/import.svg').default, + manualImport: require('./icons/file/manualImport.svg').default, + indexImport: require('./icons/file/indexImport.svg').default, + csvImport: require('./icons/file/csv.svg').default, + qaImport: require('./icons/file/qaImport.svg').default, + uploadFile: require('./icons/file/uploadFile.svg').default, + closeLight: require('./icons/light/close.svg').default }; export type IconName = keyof typeof map; diff --git a/client/src/components/Radio/index.tsx b/client/src/components/Radio/index.tsx index 48ba742bf..a3c056885 100644 --- a/client/src/components/Radio/index.tsx +++ b/client/src/components/Radio/index.tsx @@ -1,25 +1,47 @@ import React from 'react'; -import { Stack, Box, Flex, useTheme } from '@chakra-ui/react'; +import { Box, Flex, useTheme, Grid, type GridProps, theme } from '@chakra-ui/react'; import type { StackProps } from '@chakra-ui/react'; +import MyIcon from '@/components/Icon'; // @ts-ignore -interface Props extends StackProps { - list: { label: string; value: string | number }[]; +interface Props extends GridProps { + list: { icon?: string; title: string; desc?: string; value: string | number }[]; + align?: 'top' | 'center'; value: string | number; onChange: (e: string | number) => void; } -const Radio = ({ list, value, onChange, ...props }: Props) => { +const MyRadio = ({ list, value, align = 'center', onChange, ...props }: Props) => { + const theme = useTheme(); return ( - + {list.map((item) => ( { borderColor: 'myGray.200' }) }} - _hover={{ - _before: { - borderColor: 'myBlue.600' - } - }} onClick={() => onChange(item.value)} > - {item.label} + {!!item.icon && } + + {item.title} + {!!item.desc && ( + + {item.desc} + + )} + ))} - + ); }; -export default Radio; +export default MyRadio; diff --git a/client/src/constants/common.ts b/client/src/constants/common.ts index b00a72b3d..acc2c87bb 100644 --- a/client/src/constants/common.ts +++ b/client/src/constants/common.ts @@ -5,6 +5,14 @@ export enum UserAuthTypeEnum { export const PRICE_SCALE = 100000; +export const fileImgs = [ + { reg: /pdf/gi, src: '/imgs/files/pdf.svg' }, + { reg: /csv/gi, src: '/imgs/files/csv.svg' }, + { reg: /(doc|docs)/gi, src: '/imgs/files/doc.svg' }, + { reg: /txt/gi, src: '/imgs/files/txt.svg' }, + { reg: /md/gi, src: '/imgs/files/markdown.svg' } +]; + export const htmlTemplate = ` diff --git a/client/src/hooks/useConfirm.tsx b/client/src/hooks/useConfirm.tsx index 43aac7f5c..cdd6345e9 100644 --- a/client/src/hooks/useConfirm.tsx +++ b/client/src/hooks/useConfirm.tsx @@ -30,7 +30,7 @@ export const useConfirm = ({ title = '提示', content }: { title?: string; cont () => ( - + {title} diff --git a/client/src/hooks/usePagination.tsx b/client/src/hooks/usePagination.tsx index 516f085ed..a51ff450e 100644 --- a/client/src/hooks/usePagination.tsx +++ b/client/src/hooks/usePagination.tsx @@ -13,13 +13,15 @@ export const usePagination = ({ pageSize = 10, params = {}, defaultRequest = true, - type = 'button' + type = 'button', + onChange }: { api: (data: any) => any; pageSize?: number; params?: Record; defaultRequest?: boolean; type?: 'button' | 'scroll'; + onChange?: (pageNum: number) => void; }) => { const elementRef = useRef(null); const { toast } = useToast(); @@ -39,6 +41,7 @@ export const usePagination = ({ setPageNum(num); res.total !== undefined && setTotal(res.total); setData(res.data); + onChange && onChange(num); } catch (error: any) { toast({ title: error?.message || '获取数据异常', diff --git a/client/src/pages/api/openapi/kb/pushData.ts b/client/src/pages/api/openapi/kb/pushData.ts index 1b58a3097..bd8729105 100644 --- a/client/src/pages/api/openapi/kb/pushData.ts +++ b/client/src/pages/api/openapi/kb/pushData.ts @@ -8,9 +8,8 @@ import { TrainingModeEnum } from '@/constants/plugin'; import { startQueue } from '@/service/utils/tools'; import { PgClient } from '@/service/pg'; import { modelToolMap } from '@/utils/plugin'; -import { OpenAiChatEnum } from '@/constants/model'; -type DateItemType = { a: string; q: string; source?: string }; +export type DateItemType = { a: string; q: string; source?: string }; export type Props = { kbId: string; diff --git a/client/src/pages/api/openapi/kb/updateData.ts b/client/src/pages/api/openapi/kb/updateData.ts index aebe83670..acea5308f 100644 --- a/client/src/pages/api/openapi/kb/updateData.ts +++ b/client/src/pages/api/openapi/kb/updateData.ts @@ -3,28 +3,46 @@ import { jsonRes } from '@/service/response'; import { authUser } from '@/service/utils/auth'; import { PgClient } from '@/service/pg'; import { withNextCors } from '@/service/utils/tools'; +import { KB, connectToDatabase } from '@/service/mongo'; import { getVector } from '../plugin/vector'; +export type Props = { + dataId: string; + kbId: string; + a?: string; + q?: string; +}; + export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { - const { dataId, a = '', q = '' } = req.body as { dataId: string; a?: string; q?: string }; + const { dataId, a = '', q = '', kbId } = req.body as Props; if (!dataId) { throw new Error('缺少参数'); } + await connectToDatabase(); + // 凭证校验 const { userId } = await authUser({ req }); + // find model + const kb = await KB.findById(kbId, 'model'); + + if (!kb) { + throw new Error("Can't find database"); + } + // get vector - const vector = await (async () => { + const { vectors = [] } = await (async () => { if (q) { return getVector({ userId, - input: [q] + input: [q], + model: kb.model }); } - return []; + return { vectors: [[]] }; })(); // 更新 pg 内容.仅修改a，不需要更新向量。 @@ -36,7 +54,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...(q ? [ { key: 'q', value: q.replace(/'/g, '"') }, - { key: 'vector', value: `[${vector[0]}]` } + { key: 'vector', value: `[${vectors[0]}]` } ] : []) ] diff --git a/client/src/pages/api/plugins/kb/detail.ts b/client/src/pages/api/plugins/kb/detail.ts index dd3e2dbaa..9e5a6547e 100644 --- a/client/src/pages/api/plugins/kb/detail.ts +++ b/client/src/pages/api/plugins/kb/detail.ts @@ -18,13 +18,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< await connectToDatabase(); - const data = await KB.findOne( - { - _id: id, - userId - }, - '_id avatar name userId tags' - ); + const data = await KB.findOne({ + _id: id, + userId + }); if (!data) { throw new Error('kb is not exist'); @@ -36,6 +33,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< avatar: data.avatar, name: data.name, userId: data.userId, + model: data.model, tags: data.tags.join(' ') } }); diff --git a/client/src/pages/chat/components/ChatHeader.tsx b/client/src/pages/chat/components/ChatHeader.tsx index 44bbc8f44..87104815b 100644 --- a/client/src/pages/chat/components/ChatHeader.tsx +++ b/client/src/pages/chat/components/ChatHeader.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useMemo } from 'react'; import { Flex, useTheme, Box } from '@chakra-ui/react'; import { useGlobalStore } from '@/store/global'; import MyIcon from '@/components/Icon'; @@ -8,18 +8,20 @@ import ToolMenu from './ToolMenu'; import { ChatItemType } from '@/types/chat'; const ChatHeader = ({ - title, history, + appName, appAvatar, onOpenSlider }: { - title: string; history: ChatItemType[]; + appName: string; appAvatar: string; onOpenSlider: () => void; }) => { const theme = useTheme(); const { isPc } = useGlobalStore(); + const title = useMemo(() => {}, []); + return ( - {title} + {appName} - {history.length}条记录 + {history.length === 0 ? '新的对话' : `${history.length}条记录`} @@ -46,7 +48,7 @@ const ChatHeader = ({ - {title} + {appName} diff --git a/client/src/pages/chat/components/ChatHistorySlider.tsx b/client/src/pages/chat/components/ChatHistorySlider.tsx index f3c21c05e..36ad2df8d 100644 --- a/client/src/pages/chat/components/ChatHistorySlider.tsx +++ b/client/src/pages/chat/components/ChatHistorySlider.tsx @@ -77,7 +77,7 @@ const ChatHistorySlider = ({ } > - + {appName} diff --git a/client/src/pages/chat/index.tsx b/client/src/pages/chat/index.tsx index dc6a9f1cb..d26dd3403 100644 --- a/client/src/pages/chat/index.tsx +++ b/client/src/pages/chat/index.tsx @@ -288,7 +288,7 @@ const Chat = () => { {/* header */} diff --git a/client/src/pages/chat/share.tsx b/client/src/pages/chat/share.tsx index de4cc21e0..ab1452f76 100644 --- a/client/src/pages/chat/share.tsx +++ b/client/src/pages/chat/share.tsx @@ -184,7 +184,7 @@ const ShareChat = ({ shareId, historyId }: { shareId: string; historyId: string {/* header */} diff --git a/client/src/pages/kb/detail/components/DataCard.tsx b/client/src/pages/kb/detail/components/DataCard.tsx index 0c43f8da6..69c3f1456 100644 --- a/client/src/pages/kb/detail/components/DataCard.tsx +++ b/client/src/pages/kb/detail/components/DataCard.tsx @@ -1,18 +1,5 @@ import React, { useCallback, useState, useRef, useEffect } from 'react'; -import { - Box, - Card, - IconButton, - Flex, - Button, - useDisclosure, - Menu, - MenuButton, - MenuList, - MenuItem, - Input, - Grid -} from '@chakra-ui/react'; +import { Box, Card, IconButton, Flex, Button, Input, Grid } from '@chakra-ui/react'; import type { KbDataItemType } from '@/types/plugin'; import { usePagination } from '@/hooks/usePagination'; import { @@ -26,15 +13,14 @@ import { fileDownload } from '@/utils/file'; import { useMutation, useQuery } from '@tanstack/react-query'; import { useToast } from '@/hooks/useToast'; import Papa from 'papaparse'; -import dynamic from 'next/dynamic'; import InputModal, { FormData as InputDataType } from './InputDataModal'; import { debounce } from 'lodash'; import { getErrText } from '@/utils/tools'; - -const SelectFileModal = dynamic(() => import('./SelectFileModal'), { ssr: true }); -const SelectCsvModal = dynamic(() => import('./SelectCsvModal'), { ssr: true }); +import MyIcon from '@/components/Icon'; +import MyTooltip from '@/components/MyTooltip'; const DataCard = ({ kbId }: { kbId: string }) => { + const BoxRef = useRef(null); const lastSearch = useRef(''); const [searchText, setSearchText] = useState(''); const { toast } = useToast(); @@ -46,74 +32,61 @@ const DataCard = ({ kbId }: { kbId: string }) => { Pagination, total, getData, - pageNum + pageNum, + pageSize } = usePagination({ api: getKbDataList, pageSize: 24, - defaultRequest: false, params: { kbId, searchText + }, + onChange() { + if (BoxRef.current) { + BoxRef.current.scrollTop = 0; + } } }); const [editInputData, setEditInputData] = useState(); - const { - isOpen: isOpenSelectFileModal, - onOpen: onOpenSelectFileModal, - onClose: onCloseSelectFileModal - } = useDisclosure(); - const { - isOpen: isOpenSelectCsvModal, - onOpen: onOpenSelectCsvModal, - onClose: onCloseSelectCsvModal - } = useDisclosure(); - - const { data: { qaListLen = 0, vectorListLen = 0 } = {}, refetch } = useQuery( - ['getModelSplitDataList', kbId], - () => getTrainingData({ kbId, init: false }), - { + const { data: { qaListLen = 0, vectorListLen = 0 } = {}, refetch: refetchTrainingData } = + useQuery(['getModelSplitDataList', kbId], () => getTrainingData({ kbId, init: false }), { onError(err) { console.log(err); } - } - ); + }); const refetchData = useCallback( (num = pageNum) => { getData(num); - refetch(); + refetchTrainingData(); return null; }, - [getData, pageNum, refetch] + [getData, pageNum, refetchTrainingData] ); // get al data and export csv const { mutate: onclickExport, isLoading: isLoadingExport = false } = useMutation({ mutationFn: () => getExportDataList(kbId), onSuccess(res) { - try { - const text = Papa.unparse({ - fields: ['question', 'answer', 'source'], - data: res - }); - fileDownload({ - text, - type: 'text/csv', - filename: 'data.csv' - }); - toast({ - title: '导出成功，下次导出需要半小时后', - status: 'success' - }); - } catch (error) { - error; - } + const text = Papa.unparse({ + fields: ['question', 'answer', 'source'], + data: res + }); + fileDownload({ + text, + type: 'text/csv', + filename: 'data.csv' + }); + toast({ + title: '导出成功，下次导出需要半小时后', + status: 'success' + }); }, onError(err: any) { toast({ - title: typeof err === 'string' ? err : err?.message || '导出异常', + title: getErrText(err, '导出异常'), status: 'error' }); console.log(err); @@ -134,59 +107,39 @@ const DataCard = ({ kbId }: { kbId: string }) => { enabled: qaListLen > 0 || vectorListLen > 0 }); - useEffect(() => { - setSearchText(''); - getData(1); - }, [kbId]); - return ( - + 知识库数据: {total}组 - } - aria-label={'refresh'} - variant={'base'} - isLoading={isLoading} - mr={[2, 4]} - size={'sm'} - onClick={() => { - getData(pageNum); - getTrainingData({ kbId, init: true }); - }} - /> + + } + aria-label={'refresh'} + variant={'base'} + isLoading={isLoading} + mr={[2, 4]} + size={'sm'} + onClick={() => { + getData(pageNum); + getTrainingData({ kbId, init: true }); + }} + /> + - - - 导入 - - - - setEditInputData({ - a: '', - q: '' - }) - } - > - 手动输入 - - 文本/文件拆分 - csv 问答对导入 - - @@ -204,7 +157,7 @@ const DataCard = ({ kbId }: { kbId: string }) => { maxW={['60%', '300px']} size={'sm'} value={searchText} - placeholder="根据匹配知识，补充知识和来源搜索" + placeholder="根据匹配知识，补充知识和来源进行搜索" onChange={(e) => { setSearchText(e.target.value); getFirstData(); @@ -245,7 +198,7 @@ const DataCard = ({ kbId }: { kbId: string }) => { } > { {item.q} - {item.a} + + {item.a} + @@ -292,9 +247,19 @@ const DataCard = ({ kbId }: { kbId: string }) => { ))} - - - + {total > pageSize && ( + + + + )} + {total === 0 && ( + + + + 知识库空空如也 + + + )} {editInputData !== undefined && ( { onSuccess={() => refetchData()} /> )} - {isOpenSelectFileModal && ( - - )} - {isOpenSelectCsvModal && ( - - )} ); }; diff --git a/client/src/pages/kb/detail/components/Import.tsx b/client/src/pages/kb/detail/components/Import.tsx new file mode 100644 index 000000000..497246bc3 --- /dev/null +++ b/client/src/pages/kb/detail/components/Import.tsx @@ -0,0 +1,83 @@ +import React, { useState } from 'react'; +import { Box, type BoxProps, Flex, Textarea, useTheme } from '@chakra-ui/react'; +import MyRadio from '@/components/Radio/index'; +import dynamic from 'next/dynamic'; + +import ManualImport from './Import/Manual'; + +const ChunkImport = dynamic(() => import('./Import/Chunk'), { + ssr: true +}); +const QAImport = dynamic(() => import('./Import/QA'), { + ssr: true +}); +const CsvImport = dynamic(() => import('./Import/Csv'), { + ssr: true +}); + +enum ImportTypeEnum { + manual = 'manual', + index = 'index', + qa = 'qa', + csv = 'csv' +} + +const ImportData = ({ kbId }: { kbId: string }) => { + const theme = useTheme(); + const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.manual); + const TitleStyle: BoxProps = { + fontWeight: 'bold', + fontSize: ['md', 'xl'], + mb: [3, 5] + }; + + return ( + + + 数据导入方式 + + + setImportType(e as `${ImportTypeEnum}`)} + /> + + + + {importType === ImportTypeEnum.manual && } + {importType === ImportTypeEnum.index && } + {importType === ImportTypeEnum.qa && } + {importType === ImportTypeEnum.csv && } + + + ); +}; + +export default ImportData; diff --git a/client/src/pages/kb/detail/components/Import/Chunk.tsx b/client/src/pages/kb/detail/components/Import/Chunk.tsx new file mode 100644 index 000000000..c36c71335 --- /dev/null +++ b/client/src/pages/kb/detail/components/Import/Chunk.tsx @@ -0,0 +1,459 @@ +import React, { useState, useCallback, useMemo } from 'react'; +import { + Box, + Flex, + Button, + useTheme, + NumberInput, + NumberInputField, + NumberInputStepper, + NumberIncrementStepper, + NumberDecrementStepper, + Image, + Textarea +} from '@chakra-ui/react'; +import { useToast } from '@/hooks/useToast'; +import { useConfirm } from '@/hooks/useConfirm'; +import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file'; +import { useMutation } from '@tanstack/react-query'; +import { postKbDataFromList } from '@/api/plugins/kb'; +import { splitText_token } from '@/utils/file'; +import { getErrText } from '@/utils/tools'; +import { formatPrice } from '@/utils/user'; +import { vectorModelList } from '@/store/static'; +import MyIcon from '@/components/Icon'; +import CloseIcon from '@/components/Icon/close'; +import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete'; +import MyTooltip from '@/components/MyTooltip'; +import { QuestionOutlineIcon } from '@chakra-ui/icons'; +import { fileImgs } from '@/constants/common'; +import { customAlphabet } from 'nanoid'; +import { TrainingModeEnum } from '@/constants/plugin'; +import FileSelect from './FileSelect'; +import { useRouter } from 'next/router'; +const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); + +const fileExtension = '.txt, .doc, .docx, .pdf, .md'; + +type FileItemType = { + id: string; + filename: string; + text: string; + icon: string; + chunks: string[]; + tokens: number; +}; + +const ChunkImport = ({ kbId }: { kbId: string }) => { + const model = vectorModelList[0]?.model; + const unitPrice = vectorModelList[0]?.price || 0.2; + const theme = useTheme(); + const router = useRouter(); + const { toast } = useToast(); + + const [chunkLen, setChunkLen] = useState(500); + const [showRePreview, setShowRePreview] = useState(false); + const [selecting, setSelecting] = useState(false); + const [files, setFiles] = useState([]); + const [previewFile, setPreviewFile] = useState(); + const [successChunks, setSuccessChunks] = useState(0); + + const totalChunk = useMemo( + () => files.reduce((sum, file) => sum + file.chunks.length, 0), + [files] + ); + const emptyFiles = useMemo(() => files.length === 0, [files]); + + // price count + const price = useMemo(() => { + return formatPrice(files.reduce((sum, file) => sum + file.tokens, 0) * unitPrice); + }, [files, unitPrice]); + + const { openConfirm, ConfirmChild } = useConfirm({ + content: `该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。` + }); + + const onSelectFile = useCallback( + async (files: File[]) => { + setSelecting(true); + try { + let promise = Promise.resolve(); + files.forEach((file) => { + promise = promise.then(async () => { + const extension = file?.name?.split('.')?.pop()?.toLowerCase(); + const icon = fileImgs.find((item) => new RegExp(item.reg).test(file.name))?.src; + const text = await (async () => { + switch (extension) { + case 'txt': + case 'md': + return readTxtContent(file); + case 'pdf': + return readPdfContent(file); + case 'doc': + case 'docx': + return readDocContent(file); + } + return ''; + })(); + + if (icon && text) { + const splitRes = splitText_token({ + text: text, + maxLen: chunkLen + }); + + setFiles((state) => [ + { + id: nanoid(), + filename: file.name, + text, + icon, + ...splitRes + }, + ...state + ]); + } + }); + }); + await promise; + } catch (error: any) { + console.log(error); + toast({ + title: typeof error === 'string' ? error : '解析文件失败', + status: 'error' + }); + } + setSelecting(false); + }, + [chunkLen, toast] + ); + + const { mutate: onclickUpload, isLoading: uploading } = useMutation({ + mutationFn: async () => { + const chunks: { a: string; q: string; source: string }[] = []; + files.forEach((file) => + file.chunks.forEach((chunk) => { + chunks.push({ + q: chunk, + a: '', + source: file.filename + }); + }) + ); + + // subsection import + let success = 0; + const step = 100; + for (let i = 0; i < chunks.length; i += step) { + const { insertLen } = await postKbDataFromList({ + kbId, + model, + data: chunks.slice(i, i + step), + mode: TrainingModeEnum.index + }); + + success += insertLen; + setSuccessChunks(success); + } + + toast({ + title: `去重后共导入 ${success} 条数据，请耐心等待训练.`, + status: 'success' + }); + + router.replace({ + query: { + kbId, + currentTab: 'data' + } + }); + }, + onError(err) { + toast({ + title: getErrText(err, '导入文件失败'), + status: 'error' + }); + } + }); + + const onRePreview = useCallback(async () => { + try { + const splitRes = files.map((item) => + splitText_token({ + text: item.text, + maxLen: chunkLen + }) + ); + + setFiles((state) => + state.map((file, index) => ({ + ...file, + ...splitRes[index] + })) + ); + setPreviewFile(undefined); + setShowRePreview(false); + } catch (error) { + toast({ + status: 'warning', + title: getErrText(error, '文本分段异常') + }); + } + }, [chunkLen, files, toast]); + + return ( + + + + + {!emptyFiles && ( + <> + + {files.map((item) => ( + setPreviewFile(item)} + > + {''} + + {item.filename} + + { + e.stopPropagation(); + setFiles((state) => state.filter((file) => file.id !== item.id)); + }} + /> + + ))} + + {/* chunk size */} + + + 段落长度 + + + + + { + setChunkLen(+e); + setShowRePreview(true); + }} + > + + + + + + + + {/* price */} + + + 预估价格 + + + + + + {} + {price}元 + + + + {showRePreview && ( + + )} + + + + )} + + {!emptyFiles && ( + + {previewFile ? ( + + + {previewFile.filename} + + setPreviewFile(undefined)} + /> + { + // @ts-ignore + const val = e.target.innerText; + setShowRePreview(true); + + setFiles((state) => + state.map((file) => + file.id === previewFile.id + ? { + ...file, + text: val + } + : file + ) + ); + }} + /> + + ) : ( + + + 分段预览({totalChunk}组) + + + {files.map((file) => + file.chunks.map((item, i) => ( + + + + # {i + 1} + + + { + setFiles((state) => + state.map((stateFile) => + stateFile.id === file.id + ? { + ...file, + chunks: [ + ...file.chunks.slice(0, i), + ...file.chunks.slice(i + 1) + ] + } + : stateFile + ) + ); + }} + /> + + { + // @ts-ignore + const val = e.target.innerText; + + if (val === '') { + setFiles((state) => + state.map((stateFile) => + stateFile.id === file.id + ? { + ...file, + chunks: [ + ...file.chunks.slice(0, i), + ...file.chunks.slice(i + 1) + ] + } + : stateFile + ) + ); + } else { + setFiles((state) => + state.map((stateFile) => + stateFile.id === file.id + ? { + ...file, + chunks: file.chunks.map((chunk, index) => + i === index ? val : chunk + ) + } + : stateFile + ) + ); + } + }} + /> + + )) + )} + + + )} + + )} + + + ); +}; + +export default ChunkImport; diff --git a/client/src/pages/kb/detail/components/Import/Csv.tsx b/client/src/pages/kb/detail/components/Import/Csv.tsx new file mode 100644 index 000000000..1fdd2818b --- /dev/null +++ b/client/src/pages/kb/detail/components/Import/Csv.tsx @@ -0,0 +1,241 @@ +import React, { useState, useCallback, useMemo } from 'react'; +import { Box, Flex, Button, useTheme, Image } from '@chakra-ui/react'; +import { useToast } from '@/hooks/useToast'; +import { useConfirm } from '@/hooks/useConfirm'; +import { useMutation } from '@tanstack/react-query'; +import { postKbDataFromList } from '@/api/plugins/kb'; +import { getErrText } from '@/utils/tools'; +import { vectorModelList } from '@/store/static'; +import MyIcon from '@/components/Icon'; +import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete'; +import { customAlphabet } from 'nanoid'; +import { TrainingModeEnum } from '@/constants/plugin'; +import FileSelect from './FileSelect'; +import { useRouter } from 'next/router'; +const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); +import { readCsvContent } from '@/utils/file'; + +const fileExtension = '.csv'; + +type FileItemType = { + id: string; + filename: string; + chunks: { q: string; a: string }[]; +}; + +const CsvImport = ({ kbId }: { kbId: string }) => { + const model = vectorModelList[0]?.model; + const theme = useTheme(); + const router = useRouter(); + const { toast } = useToast(); + + const [selecting, setSelecting] = useState(false); + const [files, setFiles] = useState([]); + const [successChunks, setSuccessChunks] = useState(0); + + const totalChunk = useMemo( + () => files.reduce((sum, file) => sum + file.chunks.length, 0), + [files] + ); + const emptyFiles = useMemo(() => files.length === 0, [files]); + + const { openConfirm, ConfirmChild } = useConfirm({ + content: `该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。` + }); + + const onSelectFile = useCallback( + async (files: File[]) => { + setSelecting(true); + try { + let promise = Promise.resolve(); + files.forEach((file) => { + promise = promise.then(async () => { + const { header, data } = await readCsvContent(file); + if (header[0] !== 'question' || header[1] !== 'answer') { + throw new Error('csv 文件格式有误'); + } + + setFiles((state) => [ + { + id: nanoid(), + filename: file.name, + chunks: data.map((item) => ({ + q: item[0], + a: item[1] + })) + }, + ...state + ]); + }); + }); + await promise; + } catch (error: any) { + console.log(error); + toast({ + title: typeof error === 'string' ? error : '解析文件失败', + status: 'error' + }); + } + setSelecting(false); + }, + [toast] + ); + + const { mutate: onclickUpload, isLoading: uploading } = useMutation({ + mutationFn: async () => { + const chunks: { a: string; q: string; source: string }[] = []; + files.forEach((file) => + file.chunks.forEach((chunk) => { + chunks.push({ + ...chunk, + source: file.filename + }); + }) + ); + + // subsection import + let success = 0; + const step = 100; + for (let i = 0; i < chunks.length; i += step) { + const { insertLen } = await postKbDataFromList({ + kbId, + model, + data: chunks.slice(i, i + step), + mode: TrainingModeEnum.index + }); + + success += insertLen; + setSuccessChunks(success); + } + + toast({ + title: `去重后共导入 ${success} 条数据，请耐心等待训练.`, + status: 'success' + }); + + router.replace({ + query: { + kbId, + currentTab: 'data' + } + }); + }, + onError(err) { + toast({ + title: getErrText(err, '导入文件失败'), + status: 'error' + }); + } + }); + + return ( + + + + + {!emptyFiles && ( + <> + + {files.map((item) => ( + + {''} + + {item.filename} + + { + e.stopPropagation(); + setFiles((state) => state.filter((file) => file.id !== item.id)); + }} + /> + + ))} + + + + + + + )} + + {!emptyFiles && ( + + + 数据预览({totalChunk}组) + + + {files.map((file) => + file.chunks.slice(0, 100).map((item, i) => ( + + + + # {i + 1} + + + { + setFiles((state) => + state.map((stateFile) => + stateFile.id === file.id + ? { + ...file, + chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)] + } + : stateFile + ) + ); + }} + /> + + + {`q: ${item.q}\na: ${item.a}`} + + + )) + )} + + + )} + + + ); +}; + +export default CsvImport; diff --git a/client/src/pages/kb/detail/components/Import/FileSelect.tsx b/client/src/pages/kb/detail/components/Import/FileSelect.tsx new file mode 100644 index 000000000..f11d5e4a1 --- /dev/null +++ b/client/src/pages/kb/detail/components/Import/FileSelect.tsx @@ -0,0 +1,49 @@ +import React from 'react'; +import { Box, Flex, type BoxProps } from '@chakra-ui/react'; +import { useLoading } from '@/hooks/useLoading'; +import { useSelectFile } from '@/hooks/useSelectFile'; + +import MyIcon from '@/components/Icon'; + +interface Props extends BoxProps { + fileExtension: string; + onSelectFile: (files: File[]) => Promise; + isLoading?: boolean; +} + +const FileSelect = ({ fileExtension, onSelectFile, isLoading, ...props }: Props) => { + const { Loading: FileSelectLoading } = useLoading(); + + const { File, onOpen } = useSelectFile({ + fileType: fileExtension, + multiple: true + }); + + return ( + + + + 拖拽文件至此，或{' '} + + 选择文件 + + + 支持 {fileExtension} 文件 + + + + ); +}; + +export default FileSelect; diff --git a/client/src/pages/kb/detail/components/Import/Manual.tsx b/client/src/pages/kb/detail/components/Import/Manual.tsx new file mode 100644 index 000000000..67a92739e --- /dev/null +++ b/client/src/pages/kb/detail/components/Import/Manual.tsx @@ -0,0 +1,100 @@ +import React, { useCallback, useState } from 'react'; +import { Box, type BoxProps, Flex, Textarea, useTheme, Button } from '@chakra-ui/react'; +import MyRadio from '@/components/Radio/index'; +import { useForm } from 'react-hook-form'; +import { useToast } from '@/hooks/useToast'; +import { useRequest } from '@/hooks/useRequest'; +import { getErrText } from '@/utils/tools'; +import { vectorModelList } from '@/store/static'; +import { postKbDataFromList } from '@/api/plugins/kb'; +import { TrainingModeEnum } from '@/constants/plugin'; + +type ManualFormType = { q: string; a: string }; + +const ManualImport = ({ kbId }: { kbId: string }) => { + const { register, handleSubmit, reset } = useForm({ + defaultValues: { q: '', a: '' } + }); + const { toast } = useToast(); + + const { mutate: onImportData, isLoading } = useRequest({ + mutationFn: async (e: ManualFormType) => { + if (e.a.length + e.q.length >= 3000) { + toast({ + title: '总长度超长了', + status: 'warning' + }); + return; + } + + try { + const data = { + a: e.a, + q: e.q, + source: '手动录入' + }; + const { insertLen } = await postKbDataFromList({ + kbId, + model: vectorModelList[0].model, + mode: TrainingModeEnum.index, + data: [data] + }); + + if (insertLen === 0) { + toast({ + title: '已存在完全一致的数据', + status: 'warning' + }); + } else { + toast({ + title: '导入数据成功,需要一段时间训练', + status: 'success' + }); + reset({ + a: '', + q: '' + }); + } + } catch (err: any) { + toast({ + title: getErrText(err, '出现了点意外~'), + status: 'error' + }); + } + } + }); + + return ( + + + + {'匹配的知识点'} +