15308555518/ FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-12-25 20:02:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

4.6.3-website dataset (#532)

This commit is contained in:
Archer 2023-12-03 20:45:57 +08:00 committed by GitHub
parent b916183848
commit a9ae270335
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
122 changed files with 3793 additions and 1360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docSite/content/docs/custom-models/chatglm2-m3e.md
View File

@ -59,7 +59,7 @@ Authorization 为 sk-aaabbbcccdddeeefffggghhhiiijjjkkk。model 为刚刚在 One
## 接入 FastGPT
修改 config.json 配置文件，在 VectorModels 中加入 chatglm2 和 M3E 模型：
修改 config.json 配置文件，在 ChatModels 中加入 chatglm2, 在 VectorModels 中加入 M3E 模型：
```json
"ChatModels": [

32
docSite/content/docs/installation/upgrading/463.md Normal file
View File

@ -0,0 +1,32 @@
---
title: 'V4.6.3(需要初始化)'
description: 'FastGPT V4.6.3'
icon: 'upgrade'
draft: false
toc: true
weight: 833
---
## 1。执行初始化 API
发起 1 个 HTTP 请求 ({{rootkey}} 替换成环境变量里的 `rootkey`，{{host}} 替换成自己域名)
1. https://xxxxx/api/admin/initv463
```bash
curl --location --request POST 'https://{{host}}/api/admin/initv463' \
--header 'rootkey: {{rootkey}}' \
--header 'Content-Type: application/json'
```
初始化说明：
1. 初始化Mongo 中 dataset，collection 和 data 的部分字段
## V4.6.3 功能介绍
1. 商业版新增 - web站点同步
2. 新增 - 集合元数据记录
3. 优化 - url 读取内容
4. 优化 - 流读取文件，防止内存溢出
5. 优化 - 4v模型自动将 url 转 base64，本地也可调试
6. 优化 - 图片压缩等级

8
packages/global/common/file/api.d.ts vendored Normal file
View File

@ -0,0 +1,8 @@
export type UrlFetchParams = {
urlList: string[];
selector?: string;
};
export type UrlFetchResponse = {
url: string;
content: string;
}[];

86
packages/global/common/file/tools.ts
View File

@ -1,3 +1,8 @@
import axios from 'axios';
import { UrlFetchParams, UrlFetchResponse } from './api.d';
import { htmlToMarkdown } from '../string/markdown';
import * as cheerio from 'cheerio';
export const formatFileSize = (bytes: number): string => {
if (bytes === 0) return '0 B';
@ -7,3 +12,84 @@ export const formatFileSize = (bytes: number): string => {
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
};
export const cheerioToHtml = ({
fetchUrl,
$,
selector
}: {
fetchUrl: string;
$: cheerio.CheerioAPI;
selector?: string;
}) => {
// get origin url
const originUrl = new URL(fetchUrl).origin;
// remove i element
$('i,script').remove();
// remove empty a element
$('a')
.filter((i, el) => {
return $(el).text().trim() === '' && $(el).children().length === 0;
})
.remove();
// if link,img startWith /, add origin url
$('a').each((i, el) => {
const href = $(el).attr('href');
if (href && href.startsWith('/')) {
$(el).attr('href', originUrl + href);
}
});
$('img').each((i, el) => {
const src = $(el).attr('src');
if (src && src.startsWith('/')) {
$(el).attr('src', originUrl + src);
}
});
return $(selector || 'body').html();
};
export const urlsFetch = async ({
urlList,
selector
}: UrlFetchParams): Promise<UrlFetchResponse> => {
urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
const response = (
await Promise.all(
urlList.map(async (url) => {
try {
const fetchRes = await axios.get(url, {
timeout: 30000
});
const $ = cheerio.load(fetchRes.data);
const md = htmlToMarkdown(
cheerioToHtml({
fetchUrl: url,
$,
selector
})
);
return {
url,
content: md
};
} catch (error) {
console.log(error, 'fetch error');
return {
url,
content: ''
};
}
})
)
).filter((item) => item.content);
return response;
};

4
packages/global/common/plugin/types/pluginRes.d.ts vendored
View File

@ -1,4 +0,0 @@
export type FetchResultItem = {
url: string;
content: string;
};

1508
packages/global/common/string/jieba.ts Normal file
View File

File diff suppressed because it is too large Load Diff

97
packages/global/common/string/markdown.ts Normal file
View File

@ -0,0 +1,97 @@
import { simpleText } from './tools';
import { NodeHtmlMarkdown } from 'node-html-markdown';
/* Delete redundant text in markdown */
export const simpleMarkdownText = (rawText: string) => {
rawText = simpleText(rawText);
// Remove a line feed from a hyperlink or picture
rawText = rawText.replace(/\[([^\]]+)\]\((.+?)\)/g, (match, linkText, url) => {
const cleanedLinkText = linkText.replace(/\n/g, ' ').trim();
if (!url) {
return '';
}
return `[${cleanedLinkText}](${url})`;
});
// replace special \.* ……
const reg1 = /\\([-.!`_(){}\[\]])/g;
if (reg1.test(rawText)) {
rawText = rawText.replace(/\\([`!*()+-_\[\]{}\\.])/g, '$1');
}
// replace \\n
rawText = rawText.replace(/\\\\n/g, '\\n');
// Remove headings and code blocks front spaces
['####', '###', '##', '#', '```', '~~~'].forEach((item) => {
const reg = new RegExp(`\\n\\s*${item}`, 'g');
if (reg.test(rawText)) {
rawText = rawText.replace(new RegExp(`\\n\\s*(${item})`, 'g'), '\n$1');
}
});
return rawText.trim();
};
/* html string to markdown */
export const htmlToMarkdown = (html?: string | null) => {
if (!html) return '';
const surround = (source: string, surroundStr: string) => `${surroundStr}${source}${surroundStr}`;
const nhm = new NodeHtmlMarkdown(
{
codeFence: '```',
codeBlockStyle: 'fenced',
ignore: ['i', 'script']
},
{
code: ({ node, parent, options: { codeFence, codeBlockStyle }, visitor }) => {
const isCodeBlock = ['PRE', 'WRAPPED-PRE'].includes(parent?.tagName!);
if (!isCodeBlock) {
return {
spaceIfRepeatingChar: true,
noEscape: true,
postprocess: ({ content }) => {
// Find longest occurring sequence of running backticks and add one more (so content is escaped)
const delimiter =
'`' + (content.match(/`+/g)?.sort((a, b) => b.length - a.length)?.[0] || '');
const padding = delimiter.length > 1 ? ' ' : '';
return surround(surround(content, padding), delimiter);
}
};
}
/* Handle code block */
if (codeBlockStyle === 'fenced') {
const language =
node.getAttribute('class')?.match(/language-(\S+)/)?.[1] ||
parent?.getAttribute('class')?.match(/language-(\S+)/)?.[1] ||
'';
return {
noEscape: true,
prefix: `${codeFence}${language}\n`,
postfix: `\n${codeFence}\n`,
childTranslators: visitor.instance.codeBlockTranslators
};
}
return {
noEscape: true,
postprocess: ({ content }) => content.replace(/^/gm, ' '),
childTranslators: visitor.instance.codeBlockTranslators
};
}
}
);
const markdown = nhm.translate(html).trim();
return simpleMarkdownText(markdown);
};

72
packages/global/common/string/textSplitter.ts
View File

@ -15,11 +15,18 @@ export const splitText2Chunks = (props: {
}): {
chunks: string[];
tokens: number;
overlapRatio?: number;
} => {
const { text = '', chunkLen, overlapRatio = 0.2 } = props;
let { text = '', chunkLen, overlapRatio = 0.2 } = props;
const splitMarker = 'SPLIT_HERE_SPLIT_HERE';
const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER';
const overlapLen = Math.round(chunkLen * overlapRatio);
// replace code block all \n to codeBlockMarker
text = text.replace(/(```[\s\S]*?```|~~~[\s\S]*?~~~)/g, function (match) {
return match.replace(/\n/g, codeBlockMarker);
});
// The larger maxLen is, the next sentence is less likely to trigger splitting
const stepReges: { reg: RegExp; maxLen: number }[] = [
{ reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
@ -27,8 +34,8 @@ export const splitText2Chunks = (props: {
{ reg: /^(###\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
{ reg: /^(####\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
{ reg: /([\n]{2})/g, maxLen: chunkLen * 1.4 },
{ reg: /([\n](?![\*\-|>`0-9]))/g, maxLen: chunkLen * 1.8 }, // (?![\*\-|>`0-9]): markdown special char
{ reg: /([\n](`))/g, maxLen: chunkLen * 4 }, // code block
{ reg: /([\n](?![\*\-|>0-9]))/g, maxLen: chunkLen * 1.8 }, // (?![\*\-|>`0-9]): markdown special char
{ reg: /([\n])/g, maxLen: chunkLen * 1.4 },
{ reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkLen * 1.4 },
@ -38,9 +45,15 @@ export const splitText2Chunks = (props: {
{ reg: /([，]|,\s)/g, maxLen: chunkLen * 2 }
];
// if use markdown title split, Separate record title title
const getSplitTexts = ({ text, step }: { text: string; step: number }) => {
if (step >= stepReges.length) {
return [text];
return [
{
text,
title: ''
}
];
}
const isMarkdownSplit = step <= 3;
const { reg } = stepReges[step];
@ -49,7 +62,17 @@ export const splitText2Chunks = (props: {
.replace(reg, isMarkdownSplit ? `${splitMarker}$1` : `$1${splitMarker}`)
.split(`${splitMarker}`)
.filter((part) => part.trim());
return splitTexts;
return splitTexts
.map((text) => {
const matchTitle = isMarkdownSplit ? text.match(reg)?.[0] || '' : '';
return {
text: isMarkdownSplit ? text.replace(matchTitle, '') : text,
title: matchTitle
};
})
.filter((item) => item.text.trim());
};
const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
@ -63,7 +86,7 @@ export const splitText2Chunks = (props: {
let overlayText = '';
for (let i = splitTexts.length - 1; i >= 0; i--) {
const currentText = splitTexts[i];
const currentText = splitTexts[i].text;
const newText = currentText + overlayText;
const newTextLen = newText.length;
@ -83,12 +106,16 @@ export const splitText2Chunks = (props: {
const splitTextRecursively = ({
text = '',
step,
lastText
lastText,
mdTitle = ''
}: {
text: string;
step: number;
lastText: string;
mdTitle: string;
}): string[] => {
const isMarkdownSplit = step <= 3;
// mini text
if (text.length <= chunkLen) {
return [text];
@ -102,7 +129,7 @@ export const splitText2Chunks = (props: {
// use slice-chunkLen to split text
const chunks: string[] = [];
for (let i = 0; i < text.length; i += chunkLen - overlapLen) {
chunks.push(text.slice(i, i + chunkLen));
chunks.push(`${mdTitle}${text.slice(i, i + chunkLen)}`);
}
return chunks;
}
@ -115,7 +142,10 @@ export const splitText2Chunks = (props: {
const chunks: string[] = [];
for (let i = 0; i < splitTexts.length; i++) {
const currentText = splitTexts[i];
const item = splitTexts[i];
const currentTitle = `${mdTitle}${item.title}`;
const currentText = item.text;
const currentTextLen = currentText.length;
const lastTextLen = lastText.length;
const newText = lastText + currentText;
@ -125,9 +155,10 @@ export const splitText2Chunks = (props: {
if (newTextLen > maxLen) {
// lastText greater minChunkLen, direct push it to chunks, not add to next chunk. (large lastText)
if (lastTextLen > minChunkLen) {
chunks.push(lastText);
chunks.push(`${currentTitle}${lastText}`);
lastText = getOneTextOverlapText({ text: lastText, step }); // next chunk will start with overlayText
i--;
continue;
}
@ -135,11 +166,12 @@ export const splitText2Chunks = (props: {
const innerChunks = splitTextRecursively({
text: newText,
step: step + 1,
lastText: ''
lastText: '',
mdTitle: currentTitle
});
const lastChunk = innerChunks[innerChunks.length - 1];
// last chunk is too small, concat it to lastText
if (lastChunk.length < minChunkLen) {
if (!isMarkdownSplit && lastChunk.length < minChunkLen) {
chunks.push(...innerChunks.slice(0, -1));
lastText = lastChunk;
} else {
@ -156,10 +188,11 @@ export const splitText2Chunks = (props: {
// size less than chunkLen, push text to last chunk. now, text definitely less than maxLen
lastText = newText;
// If the chunk size reaches, add a chunk
if (newTextLen >= chunkLen) {
chunks.push(lastText);
lastText = getOneTextOverlapText({ text: lastText, step });
// markdown paragraph block: Direct addition; If the chunk size reaches, add a chunk
if (isMarkdownSplit || newTextLen >= chunkLen) {
chunks.push(`${currentTitle}${lastText}`);
lastText = isMarkdownSplit ? '' : getOneTextOverlapText({ text: lastText, step });
}
}
@ -168,7 +201,7 @@ export const splitText2Chunks = (props: {
if (lastText.length < chunkLen * 0.4) {
chunks[chunks.length - 1] = chunks[chunks.length - 1] + lastText;
} else {
chunks.push(lastText);
chunks.push(`${mdTitle}${lastText}`);
}
}
@ -179,8 +212,9 @@ export const splitText2Chunks = (props: {
const chunks = splitTextRecursively({
text,
step: 0,
lastText: ''
});
lastText: '',
mdTitle: ''
}).map((chunk) => chunk.replaceAll(codeBlockMarker, '\n')); // restore code block
const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0);

6
packages/global/common/system/utils.ts Normal file
View File

@ -0,0 +1,6 @@
export const delay = (ms: number) =>
new Promise((resolve) => {
setTimeout(() => {
resolve('');
}, ms);
});

2
packages/global/core/chat/constants.ts
View File

@ -5,7 +5,6 @@ export enum ChatRoleEnum {
Function = 'Function',
Tool = 'Tool'
}
export const ChatRoleMap = {
[ChatRoleEnum.System]: {
name: '系统提示词'
@ -30,7 +29,6 @@ export enum ChatSourceEnum {
share = 'share',
api = 'api'
}
export const ChatSourceMap = {
[ChatSourceEnum.test]: {
name: 'chat.logs.test'

31
packages/global/core/dataset/api.d.ts vendored
View File

@ -1,8 +1,32 @@
import { DatasetDataIndexItemType } from './type';
import { DatasetDataIndexItemType, DatasetSchemaType } from './type';
import { DatasetCollectionTrainingModeEnum, DatasetCollectionTypeEnum } from './constant';
import type { LLMModelItemType } from '../ai/model.d';
/* ================= dataset ===================== */
export type DatasetUpdateBody = {
id: string;
parentId?: string;
tags?: string[];
name?: string;
avatar?: string;
permission?: DatasetSchemaType['permission'];
agentModel?: LLMModelItemType;
websiteConfig?: DatasetSchemaType['websiteConfig'];
status?: DatasetSchemaType['status'];
};
/* ================= collection ===================== */
export type CreateDatasetCollectionParams = {
datasetId: string;
parentId?: string;
name: string;
type: `${DatasetCollectionTypeEnum}`;
trainingType?: `${DatasetCollectionTrainingModeEnum}`;
chunkSize?: number;
fileId?: string;
rawLink?: string;
metadata?: Record<string, any>;
};
/* ================= data ===================== */
export type PgSearchRawType = {
@ -18,3 +42,8 @@ export type PushDatasetDataChunkProps = {
a?: string; // bonus content
indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
};
export type PostWebsiteSyncParams = {
datasetId: string;
billId: string;
};

32
packages/global/core/dataset/constant.ts
View File

@ -3,15 +3,37 @@ export const PgDatasetTableName = 'modeldata';
/* ------------ dataset -------------- */
export enum DatasetTypeEnum {
folder = 'folder',
dataset = 'dataset'
dataset = 'dataset',
websiteDataset = 'websiteDataset' // depp link
}
export const DatasetTypeMap = {
[DatasetTypeEnum.folder]: {
name: 'folder'
icon: 'core/dataset/folderDataset',
label: 'core.dataset.Folder Dataset',
collectionLabel: 'common.Folder'
},
[DatasetTypeEnum.dataset]: {
name: 'dataset'
icon: 'core/dataset/commonDataset',
label: 'core.dataset.Common Dataset',
collectionLabel: 'common.File'
},
[DatasetTypeEnum.websiteDataset]: {
icon: 'core/dataset/websiteDataset',
label: 'core.dataset.Website Dataset',
collectionLabel: 'common.Website'
}
};
export enum DatasetStatusEnum {
active = 'active',
syncing = 'syncing'
}
export const DatasetStatusMap = {
[DatasetStatusEnum.active]: {
label: 'core.dataset.status.active'
},
[DatasetStatusEnum.syncing]: {
label: 'core.dataset.status.syncing'
}
};
@ -19,7 +41,7 @@ export const DatasetTypeMap = {
export enum DatasetCollectionTypeEnum {
folder = 'folder',
file = 'file',
link = 'link',
link = 'link', // one link
virtual = 'virtual'
}
export const DatasetCollectionTypeMap = {

21
packages/global/core/dataset/type.d.ts vendored
View File

@ -4,6 +4,7 @@ import { PushDatasetDataChunkProps } from './api';
import {
DatasetCollectionTypeEnum,
DatasetDataIndexTypeEnum,
DatasetStatusEnum,
DatasetTypeEnum,
TrainingModeEnum
} from './constant';
@ -20,9 +21,14 @@ export type DatasetSchemaType = {
name: string;
vectorModel: string;
agentModel: string;
tags: string[];
intro: string;
type: `${DatasetTypeEnum}`;
status: `${DatasetStatusEnum}`;
permission: `${PermissionTypeEnum}`;
websiteConfig?: {
url: string;
selector: string;
};
};
export type DatasetCollectionSchemaType = {
@ -39,6 +45,7 @@ export type DatasetCollectionSchemaType = {
chunkSize: number;
fileId?: string;
rawLink?: string;
metadata?: Record<string, any>;
};
export type DatasetDataIndexItemType = {
@ -91,6 +98,18 @@ export type DatasetDataWithCollectionType = Omit<DatasetDataSchemaType, 'collect
};
/* ================= dataset ===================== */
export type DatasetListItemType = {
_id: string;
parentId: string;
avatar: string;
name: string;
intro: string;
type: `${DatasetTypeEnum}`;
isOwner: boolean;
canWrite: boolean;
permission: `${PermissionTypeEnum}`;
vectorModel: VectorModelItemType;
};
export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel'> & {
vectorModel: VectorModelItemType;
agentModel: LLMModelItemType;

9
packages/global/package.json
View File

@ -3,13 +3,16 @@
"version": "1.0.0",
"dependencies": {
"axios": "^1.5.1",
"timezones-list": "^3.0.2",
"cheerio": "1.0.0-rc.12",
"dayjs": "^1.11.7",
"encoding": "^0.1.13",
"js-tiktoken": "^1.0.7",
"node-html-markdown": "^1.3.0",
"openai": "^4.16.1",
"js-tiktoken": "^1.0.7"
"timezones-list": "^3.0.2"
},
"devDependencies": {
"@types/node": "^20.8.5"
"@types/node": "^20.8.5",
"@types/turndown": "^5.0.4"
}
}

15
packages/global/support/wallet/bill/tools.ts
View File

@ -1,6 +1,7 @@
/* bill common */
import { PRICE_SCALE } from './constants';
import { BillItemType, BillSchema } from './type';
import { BillSourceEnum } from './constants';
import { AuthUserTypeEnum } from '../../permission/constant';
/**
* dataset price / PRICE_SCALE = real price
@ -8,3 +9,15 @@ import { BillItemType, BillSchema } from './type';
export const formatPrice = (val = 0, multiple = 1) => {
return Number(((val / PRICE_SCALE) * multiple).toFixed(10));
};
export const getBillSourceByAuthType = ({
shareId,
authType
}: {
shareId?: string;
authType?: `${AuthUserTypeEnum}`;
}) => {
if (shareId) return BillSourceEnum.shareLink;
if (authType === AuthUserTypeEnum.apikey) return BillSourceEnum.api;
return BillSourceEnum.fastgpt;
};

8
packages/service/common/api/plusRequest.ts
View File

@ -101,18 +101,18 @@ export function request(url: string, data: any, config: ConfigType, method: Meth
* @param {Object} config
* @returns
*/
export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
export function GET<T = undefined>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
return request(url, params, config, 'GET');
}
export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
export function POST<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
return request(url, data, config, 'POST');
}
export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
export function PUT<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
return request(url, data, config, 'PUT');
}
export function DELETE<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
export function DELETE<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
return request(url, data, config, 'DELETE');
}

13
packages/service/common/file/gridfs/controller.ts
View File

@ -89,7 +89,7 @@ export async function delFileById({
return true;
}
export async function getDownloadBuf({
export async function getDownloadStream({
bucketName,
fileId
}: {
@ -98,14 +98,5 @@ export async function getDownloadBuf({
}) {
const bucket = getGridBucket(bucketName);
const stream = bucket.openDownloadStream(new Types.ObjectId(fileId));
const buf: Buffer = await new Promise((resolve, reject) => {
const buffers: Buffer[] = [];
stream.on('data', (data) => buffers.push(data));
stream.on('error', reject);
stream.on('end', () => resolve(Buffer.concat(buffers)));
});
return buf;
return bucket.openDownloadStream(new Types.ObjectId(fileId));
}

20
packages/service/core/chat/utils.ts
View File

@ -3,6 +3,7 @@ import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants
import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
import axios from 'axios';
/* slice chat context by tokens */
export function ChatContextFilter({
@ -81,11 +82,13 @@ export function ChatContextFilter({
}
]
*/
export function formatStr2ChatContent(str: string) {
export async function formatStr2ChatContent(str: string) {
const content: ChatCompletionContentPart[] = [];
let lastIndex = 0;
const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
const imgKey: 'image_url' = 'image_url';
let match;
while ((match = regex.exec(str)) !== null) {
@ -115,7 +118,7 @@ export function formatStr2ChatContent(str: string) {
content.push(
...jsonLines.map((item) => ({
type: 'image_url' as any,
type: imgKey,
image_url: {
url: item.src
}
@ -148,5 +151,18 @@ export function formatStr2ChatContent(str: string) {
if (content.length === 1 && content[0].type === 'text') {
return content[0].text;
}
if (!content) return null;
// load img to base64
for await (const item of content) {
if (item.type === imgKey && item[imgKey]?.url) {
const response = await axios.get(item[imgKey].url, {
responseType: 'arraybuffer'
});
const base64 = Buffer.from(response.data).toString('base64');
item[imgKey].url = `data:${response.headers['content-type']};base64,${base64}`;
}
}
return content ? content : null;
}

73
packages/service/core/dataset/collection/controller.ts Normal file
View File

@ -0,0 +1,73 @@
import {
DatasetCollectionTrainingModeEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constant';
import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { MongoDatasetCollection } from './schema';
export async function createOneCollection({
name,
parentId,
datasetId,
type,
trainingType = DatasetCollectionTrainingModeEnum.manual,
chunkSize = 0,
fileId,
rawLink,
teamId,
tmbId,
metadata = {}
}: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
const { _id } = await MongoDatasetCollection.create({
name,
teamId,
tmbId,
datasetId,
parentId: parentId || null,
type,
trainingType,
chunkSize,
fileId,
rawLink,
metadata
});
// create default collection
if (type === DatasetCollectionTypeEnum.folder) {
await createDefaultCollection({
datasetId,
parentId: _id,
teamId,
tmbId
});
}
return _id;
}
// create default collection
export function createDefaultCollection({
name = '手动录入',
datasetId,
parentId,
teamId,
tmbId
}: {
name?: '手动录入' | '手动标注';
datasetId: string;
parentId?: string;
teamId: string;
tmbId: string;
}) {
return MongoDatasetCollection.create({
name,
teamId,
tmbId,
datasetId,
parentId,
type: DatasetCollectionTypeEnum.virtual,
trainingType: DatasetCollectionTrainingModeEnum.manual,
chunkSize: 0,
updateTime: new Date('2099')
});
}

9
packages/service/core/dataset/collection/schema.ts
View File

@ -39,15 +39,16 @@ const DatasetCollectionSchema = new Schema({
ref: DatasetCollectionName,
required: true
},
name: {
type: String,
required: true
},
type: {
type: String,
enum: Object.keys(DatasetCollectionTypeMap),
required: true
},
name: {
type: String,
required: true
},
createTime: {
type: Date,
default: () => new Date()

75
packages/service/core/dataset/data/controller.ts Normal file
View File

@ -0,0 +1,75 @@
import { MongoDatasetData } from './schema';
import { deletePgDataById } from './pg';
import { MongoDatasetTraining } from '../training/schema';
import { delFileById } from '../../../common/file/gridfs/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { MongoDatasetCollection } from '../collection/schema';
import { delDatasetFiles } from '../file/controller';
import { delay } from '@fastgpt/global/common/system/utils';
/* delete all data by datasetIds */
export async function delDatasetRelevantData({ datasetIds }: { datasetIds: string[] }) {
datasetIds = datasetIds.map((item) => String(item));
// delete training data(There could be a training mission)
await MongoDatasetTraining.deleteMany({
datasetId: { $in: datasetIds }
});
// delete related files
await Promise.all(datasetIds.map((id) => delDatasetFiles({ datasetId: id })));
await delay(1000);
// delete pg data
await deletePgDataById(`dataset_id IN ('${datasetIds.join("','")}')`);
// delete dataset.datas
await MongoDatasetData.deleteMany({ datasetId: { $in: datasetIds } });
// delete collections
await MongoDatasetCollection.deleteMany({
datasetId: { $in: datasetIds }
});
}
/**
* delete all data by collectionIds
*/
export async function delCollectionRelevantData({
collectionIds,
fileIds
}: {
collectionIds: string[];
fileIds: string[];
}) {
collectionIds = collectionIds.map((item) => String(item));
const filterFileIds = fileIds.filter(Boolean);
// delete training data
await MongoDatasetTraining.deleteMany({
collectionId: { $in: collectionIds }
});
// delete file
await Promise.all(
filterFileIds.map((fileId) => {
return delFileById({
bucketName: BucketNameEnum.dataset,
fileId
});
})
);
await delay(1000);
// delete pg data
await deletePgDataById(`collection_id IN ('${collectionIds.join("','")}')`);
// delete dataset.datas
await MongoDatasetData.deleteMany({ collectionId: { $in: collectionIds } });
}
/**
* delete one data by mongoDataId
*/
export async function delDatasetDataByDataId(mongoDataId: string) {
await deletePgDataById(['data_id', mongoDataId]);
await MongoDatasetData.findByIdAndDelete(mongoDataId);
}

28
packages/service/core/dataset/data/pg.ts Normal file
View File

@ -0,0 +1,28 @@
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '../../../common/pg';
export async function deletePgDataById(
where: ['id' | 'dataset_id' | 'collection_id' | 'data_id', string] | string
) {
let retry = 2;
async function deleteData(): Promise<any> {
try {
await PgClient.delete(PgDatasetTableName, {
where: [where]
});
} catch (error) {
if (--retry < 0) {
return Promise.reject(error);
}
await delay(500);
return deleteData();
}
}
await deleteData();
return {
tokenLen: 0
};
}

5
packages/service/core/dataset/data/schema.ts
View File

@ -79,6 +79,9 @@ const DatasetDataSchema = new Schema({
chunkIndex: {
type: Number,
default: 0
},
inited: {
type: Boolean
}
});
@ -88,7 +91,7 @@ try {
DatasetDataSchema.index({ collectionId: 1 });
// full text index
DatasetDataSchema.index({ datasetId: 1, fullTextToken: 'text' });
DatasetDataSchema.index({ fullTextToken: 1 });
DatasetDataSchema.index({ inited: 1 });
} catch (error) {
console.log(error);
}

49
packages/service/core/dataset/schema.ts
View File

@ -1,7 +1,11 @@
import { connectionMongo, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constant';
import {
DatasetStatusEnum,
DatasetStatusMap,
DatasetTypeMap
} from '@fastgpt/global/core/dataset/constant';
import {
TeamCollectionName,
TeamMemberCollectionName
@ -31,9 +35,16 @@ const DatasetSchema = new Schema({
ref: TeamMemberCollectionName,
required: true
},
updateTime: {
type: Date,
default: () => new Date()
type: {
type: String,
enum: Object.keys(DatasetTypeMap),
required: true,
default: 'dataset'
},
status: {
type: String,
enum: Object.keys(DatasetStatusMap),
default: DatasetStatusEnum.active
},
avatar: {
type: String,
@ -43,6 +54,10 @@ const DatasetSchema = new Schema({
type: String,
required: true
},
updateTime: {
type: Date,
default: () => new Date()
},
vectorModel: {
type: String,
required: true,
@ -53,24 +68,26 @@ const DatasetSchema = new Schema({
required: true,
default: 'gpt-3.5-turbo-16k'
},
type: {
intro: {
type: String,
enum: Object.keys(DatasetTypeMap),
required: true,
default: 'dataset'
},
tags: {
type: [String],
default: [],
set(val: string | string[]) {
if (Array.isArray(val)) return val;
return val.split(' ').filter((item) => item);
}
default: ''
},
permission: {
type: String,
enum: Object.keys(PermissionTypeMap),
default: PermissionTypeEnum.private
},
websiteConfig: {
type: {
url: {
type: String,
required: true
},
selector: {
type: String,
default: 'body'
}
}
}
});

446
pnpm-lock.yaml
View File

@ -1,4 +1,4 @@
lockfileVersion: '6.0'
lockfileVersion: '6.1'
settings:
autoInstallPeers: true
@ -45,6 +45,9 @@ importers:
axios:
specifier: ^1.5.1
version: registry.npmmirror.com/axios@1.5.1
cheerio:
specifier: 1.0.0-rc.12
version: registry.npmmirror.com/cheerio@1.0.0-rc.12
dayjs:
specifier: ^1.11.7
version: registry.npmmirror.com/dayjs@1.11.10
@ -54,6 +57,9 @@ importers:
js-tiktoken:
specifier: ^1.0.7
version: registry.npmmirror.com/js-tiktoken@1.0.7
node-html-markdown:
specifier: ^1.3.0
version: registry.npmmirror.com/node-html-markdown@1.3.0
openai:
specifier: ^4.16.1
version: registry.npmmirror.com/openai@4.16.1(encoding@0.1.13)
@ -64,6 +70,9 @@ importers:
'@types/node':
specifier: ^20.8.5
version: registry.npmmirror.com/@types/node@20.8.7
'@types/turndown':
specifier: ^5.0.4
version: registry.npmmirror.com/@types/turndown@5.0.4
packages/service:
dependencies:
@ -161,9 +170,6 @@ importers:
'@fastgpt/web':
specifier: workspace:*
version: link:../../packages/web
'@mozilla/readability':
specifier: ^0.4.4
version: registry.npmmirror.com/@mozilla/readability@0.4.4
'@node-rs/jieba':
specifier: ^1.7.2
version: registry.npmmirror.com/@node-rs/jieba@1.7.2
@ -209,9 +215,6 @@ importers:
jschardet:
specifier: ^3.0.0
version: registry.npmmirror.com/jschardet@3.0.0
jsdom:
specifier: ^22.1.0
version: registry.npmmirror.com/jsdom@22.1.0
jsonwebtoken:
specifier: ^9.0.2
version: registry.npmmirror.com/jsonwebtoken@9.0.2
@ -300,9 +303,6 @@ importers:
'@types/js-cookie':
specifier: ^3.0.3
version: registry.npmmirror.com/@types/js-cookie@3.0.5
'@types/jsdom':
specifier: ^21.1.1
version: registry.npmmirror.com/@types/jsdom@21.1.4
'@types/jsonwebtoken':
specifier: ^9.0.3
version: registry.npmmirror.com/@types/jsonwebtoken@9.0.4
@ -3575,13 +3575,6 @@ packages:
dev: false
optional: true
registry.npmmirror.com/@mozilla/readability@0.4.4:
resolution: {integrity: sha512-MCgZyANpJ6msfvVMi6+A0UAsvZj//4OHREYUB9f2087uXHVoU+H+SWhuihvb1beKpM323bReQPRio0WNk2+V6g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@mozilla/readability/-/readability-0.4.4.tgz}
name: '@mozilla/readability'
version: 0.4.4
engines: {node: '>=14.0.0'}
dev: false
registry.npmmirror.com/@next/env@13.5.2:
resolution: {integrity: sha512-dUseBIQVax+XtdJPzhwww4GetTjlkRSsXeQnisIJWBaHsnxYcN2RGzsPHi58D6qnkATjnhuAtQTJmR1hKYQQPg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@next/env/-/env-13.5.2.tgz}
name: '@next/env'
@ -4248,13 +4241,6 @@ packages:
use-sync-external-store: registry.npmmirror.com/use-sync-external-store@1.2.0(react@18.2.0)
dev: false
registry.npmmirror.com/@tootallnate/once@2.0.0:
resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@tootallnate/once/-/once-2.0.0.tgz}
name: '@tootallnate/once'
version: 2.0.0
engines: {node: '>= 10'}
dev: false
registry.npmmirror.com/@trysound/sax@0.2.0:
resolution: {integrity: sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@trysound/sax/-/sax-0.2.0.tgz}
name: '@trysound/sax'
@ -4618,16 +4604,6 @@ packages:
version: 3.0.5
dev: true
registry.npmmirror.com/@types/jsdom@21.1.4:
resolution: {integrity: sha512-NzAMLEV0KQ4cBaDx3Ls8VfJUElyDUm1xrtYRmcMK0gF8L5xYbujFVaQlJ50yinQ/d47j2rEP1XUzkiYrw4YRFA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/jsdom/-/jsdom-21.1.4.tgz}
name: '@types/jsdom'
version: 21.1.4
dependencies:
'@types/node': registry.npmmirror.com/@types/node@20.8.7
'@types/tough-cookie': registry.npmmirror.com/@types/tough-cookie@4.0.4
parse5: registry.npmmirror.com/parse5@7.1.2
dev: true
registry.npmmirror.com/@types/json5@0.0.29:
resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/json5/-/json5-0.0.29.tgz}
name: '@types/json5'
@ -4826,12 +4802,6 @@ packages:
'@types/node': registry.npmmirror.com/@types/node@20.8.7
dev: true
registry.npmmirror.com/@types/tough-cookie@4.0.4:
resolution: {integrity: sha512-95Sfz4nvMAb0Nl9DTxN3j64adfwfbBPEYq14VN7zT5J5O2M9V6iZMIIQU1U+pJyl9agHYHNCqhCXgyEtIRRa5A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/tough-cookie/-/tough-cookie-4.0.4.tgz}
name: '@types/tough-cookie'
version: 4.0.4
dev: true
registry.npmmirror.com/@types/triple-beam@1.3.4:
resolution: {integrity: sha512-HlJjF3wxV4R2VQkFpKe0YqJLilYNgtRtsqqZtby7RkVsSs+i+vbyzjtUwpFEdUCKcrGzCiEJE7F/0mKjh0sunA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/triple-beam/-/triple-beam-1.3.4.tgz}
name: '@types/triple-beam'
@ -4846,6 +4816,12 @@ packages:
'@types/node': registry.npmmirror.com/@types/node@20.8.7
dev: true
registry.npmmirror.com/@types/turndown@5.0.4:
resolution: {integrity: sha512-28GI33lCCkU4SGH1GvjDhFgOVr+Tym4PXGBIU1buJUa6xQolniPArtUT+kv42RR2N9MsMLInkr904Aq+ESHBJg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/turndown/-/turndown-5.0.4.tgz}
name: '@types/turndown'
version: 5.0.4
dev: true
registry.npmmirror.com/@types/unist@2.0.9:
resolution: {integrity: sha512-zC0iXxAv1C1ERURduJueYzkzZ2zaGyc+P2c95hgkikHPr3z8EdUZOlgEQ5X0DRmwDZn+hekycQnoeiiRVrmilQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/unist/-/unist-2.0.9.tgz}
name: '@types/unist'
@ -4967,12 +4943,6 @@ packages:
'@zag-js/dom-query': registry.npmmirror.com/@zag-js/dom-query@0.16.0
dev: false
registry.npmmirror.com/abab@2.0.6:
resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/abab/-/abab-2.0.6.tgz}
name: abab
version: 2.0.6
dev: false
registry.npmmirror.com/abort-controller@3.0.0:
resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/abort-controller/-/abort-controller-3.0.0.tgz}
name: abort-controller
@ -5001,17 +4971,6 @@ packages:
hasBin: true
dev: true
registry.npmmirror.com/agent-base@6.0.2:
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/agent-base/-/agent-base-6.0.2.tgz}
name: agent-base
version: 6.0.2
engines: {node: '>= 6.0.0'}
dependencies:
debug: registry.npmmirror.com/debug@4.3.4
transitivePeerDependencies:
- supports-color
dev: false
registry.npmmirror.com/agentkeepalive@4.5.0:
resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz}
name: agentkeepalive
@ -5435,7 +5394,6 @@ packages:
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz}
name: boolbase
version: 1.0.0
dev: true
registry.npmmirror.com/brace-expansion@1.1.11:
resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz}
@ -5720,6 +5678,34 @@ packages:
get-func-name: registry.npmmirror.com/get-func-name@2.0.2
dev: true
registry.npmmirror.com/cheerio-select@2.1.0:
resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cheerio-select/-/cheerio-select-2.1.0.tgz}
name: cheerio-select
version: 2.1.0
dependencies:
boolbase: registry.npmmirror.com/boolbase@1.0.0
css-select: registry.npmmirror.com/css-select@5.1.0
css-what: registry.npmmirror.com/css-what@6.1.0
domelementtype: registry.npmmirror.com/domelementtype@2.3.0
domhandler: registry.npmmirror.com/domhandler@5.0.3
domutils: registry.npmmirror.com/domutils@3.1.0
dev: false
registry.npmmirror.com/cheerio@1.0.0-rc.12:
resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cheerio/-/cheerio-1.0.0-rc.12.tgz}
name: cheerio
version: 1.0.0-rc.12
engines: {node: '>= 6'}
dependencies:
cheerio-select: registry.npmmirror.com/cheerio-select@2.1.0
dom-serializer: registry.npmmirror.com/dom-serializer@2.0.0
domhandler: registry.npmmirror.com/domhandler@5.0.3
domutils: registry.npmmirror.com/domutils@3.1.0
htmlparser2: registry.npmmirror.com/htmlparser2@8.0.2
parse5: registry.npmmirror.com/parse5@7.1.2
parse5-htmlparser2-tree-adapter: registry.npmmirror.com/parse5-htmlparser2-tree-adapter@7.0.0
dev: false
registry.npmmirror.com/chokidar@3.5.3:
resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz}
name: chokidar
@ -6108,6 +6094,18 @@ packages:
nth-check: registry.npmmirror.com/nth-check@2.1.1
dev: true
registry.npmmirror.com/css-select@5.1.0:
resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/css-select/-/css-select-5.1.0.tgz}
name: css-select
version: 5.1.0
dependencies:
boolbase: registry.npmmirror.com/boolbase@1.0.0
css-what: registry.npmmirror.com/css-what@6.1.0
domhandler: registry.npmmirror.com/domhandler@5.0.3
domutils: registry.npmmirror.com/domutils@3.1.0
nth-check: registry.npmmirror.com/nth-check@2.1.1
dev: false
registry.npmmirror.com/css-tree@1.1.3:
resolution: {integrity: sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/css-tree/-/css-tree-1.1.3.tgz}
name: css-tree
@ -6123,7 +6121,6 @@ packages:
name: css-what
version: 6.1.0
engines: {node: '>= 6'}
dev: true
registry.npmmirror.com/csso@4.2.0:
resolution: {integrity: sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/csso/-/csso-4.2.0.tgz}
@ -6134,15 +6131,6 @@ packages:
css-tree: registry.npmmirror.com/css-tree@1.1.3
dev: true
registry.npmmirror.com/cssstyle@3.0.0:
resolution: {integrity: sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cssstyle/-/cssstyle-3.0.0.tgz}
name: cssstyle
version: 3.0.0
engines: {node: '>=14'}
dependencies:
rrweb-cssom: registry.npmmirror.com/rrweb-cssom@0.6.0
dev: false
registry.npmmirror.com/csstype@3.1.2:
resolution: {integrity: sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/csstype/-/csstype-3.1.2.tgz}
name: csstype
@ -6539,17 +6527,6 @@ packages:
version: 1.0.8
dev: true
registry.npmmirror.com/data-urls@4.0.0:
resolution: {integrity: sha512-/mMTei/JXPqvFqQtfyTowxmJVwr2PVAeCcDxyFf6LhoOu/09TX2OX3kb2wzi4DMXcfj4OItwDOnhl5oziPnT6g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/data-urls/-/data-urls-4.0.0.tgz}
name: data-urls
version: 4.0.0
engines: {node: '>=14'}
dependencies:
abab: registry.npmmirror.com/abab@2.0.6
whatwg-mimetype: registry.npmmirror.com/whatwg-mimetype@3.0.0
whatwg-url: registry.npmmirror.com/whatwg-url@12.0.1
dev: false
registry.npmmirror.com/date-fns@2.30.0:
resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/date-fns/-/date-fns-2.30.0.tgz}
name: date-fns
@ -6591,12 +6568,6 @@ packages:
dependencies:
ms: registry.npmmirror.com/ms@2.1.2
registry.npmmirror.com/decimal.js@10.4.3:
resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/decimal.js/-/decimal.js-10.4.3.tgz}
name: decimal.js
version: 10.4.3
dev: false
registry.npmmirror.com/decode-named-character-reference@1.0.2:
resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz}
name: decode-named-character-reference
@ -6769,6 +6740,16 @@ packages:
entities: registry.npmmirror.com/entities@2.2.0
dev: true
registry.npmmirror.com/dom-serializer@2.0.0:
resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dom-serializer/-/dom-serializer-2.0.0.tgz}
name: dom-serializer
version: 2.0.0
dependencies:
domelementtype: registry.npmmirror.com/domelementtype@2.3.0
domhandler: registry.npmmirror.com/domhandler@5.0.3
entities: registry.npmmirror.com/entities@4.5.0
dev: false
registry.npmmirror.com/domain-browser@4.23.0:
resolution: {integrity: sha512-ArzcM/II1wCCujdCNyQjXrAFwS4mrLh4C7DZWlaI8mdh7h3BfKdNd3bKXITfl2PT9FtfQqaGvhi1vPRQPimjGA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domain-browser/-/domain-browser-4.23.0.tgz}
name: domain-browser
@ -6780,16 +6761,6 @@ packages:
resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz}
name: domelementtype
version: 2.3.0
dev: true
registry.npmmirror.com/domexception@4.0.0:
resolution: {integrity: sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domexception/-/domexception-4.0.0.tgz}
name: domexception
version: 4.0.0
engines: {node: '>=12'}
dependencies:
webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
dev: false
registry.npmmirror.com/domhandler@4.3.1:
resolution: {integrity: sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domhandler/-/domhandler-4.3.1.tgz}
@ -6800,6 +6771,15 @@ packages:
domelementtype: registry.npmmirror.com/domelementtype@2.3.0
dev: true
registry.npmmirror.com/domhandler@5.0.3:
resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domhandler/-/domhandler-5.0.3.tgz}
name: domhandler
version: 5.0.3
engines: {node: '>= 4'}
dependencies:
domelementtype: registry.npmmirror.com/domelementtype@2.3.0
dev: false
registry.npmmirror.com/dompurify@3.0.6:
resolution: {integrity: sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dompurify/-/dompurify-3.0.6.tgz}
name: dompurify
@ -6816,6 +6796,16 @@ packages:
domhandler: registry.npmmirror.com/domhandler@4.3.1
dev: true
registry.npmmirror.com/domutils@3.1.0:
resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domutils/-/domutils-3.1.0.tgz}
name: domutils
version: 3.1.0
dependencies:
dom-serializer: registry.npmmirror.com/dom-serializer@2.0.0
domelementtype: registry.npmmirror.com/domelementtype@2.3.0
domhandler: registry.npmmirror.com/domhandler@5.0.3
dev: false
registry.npmmirror.com/downloadjs@1.4.7:
resolution: {integrity: sha512-LN1gO7+u9xjU5oEScGFKvXhYf7Y/empUIIEAGBs1LzUq/rg5duiDrkuH5A2lQGd5jfMOb9X9usDa2oVXwJ0U/Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/downloadjs/-/downloadjs-1.4.7.tgz}
name: downloadjs
@ -8285,6 +8275,13 @@ packages:
space-separated-tokens: registry.npmmirror.com/space-separated-tokens@2.0.2
dev: false
registry.npmmirror.com/he@1.2.0:
resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/he/-/he-1.2.0.tgz}
name: he
version: 1.2.0
hasBin: true
dev: false
registry.npmmirror.com/heap@0.2.7:
resolution: {integrity: sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/heap/-/heap-0.2.7.tgz}
name: heap
@ -8321,15 +8318,6 @@ packages:
dependencies:
react-is: registry.npmmirror.com/react-is@16.13.1
registry.npmmirror.com/html-encoding-sniffer@3.0.0:
resolution: {integrity: sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz}
name: html-encoding-sniffer
version: 3.0.0
engines: {node: '>=12'}
dependencies:
whatwg-encoding: registry.npmmirror.com/whatwg-encoding@2.0.0
dev: false
registry.npmmirror.com/html-parse-stringify@3.0.1:
resolution: {integrity: sha512-KknJ50kTInJ7qIScF3jeaFRpMpE8/lfiTdzf/twXyPBLAGrLRTmkz3AdTnKeh40X8k9L2fdYwEp/42WGXIRGcg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/html-parse-stringify/-/html-parse-stringify-3.0.1.tgz}
name: html-parse-stringify
@ -8337,17 +8325,15 @@ packages:
dependencies:
void-elements: registry.npmmirror.com/void-elements@3.1.0
registry.npmmirror.com/http-proxy-agent@5.0.0:
resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz}
name: http-proxy-agent
version: 5.0.0
engines: {node: '>= 6'}
registry.npmmirror.com/htmlparser2@8.0.2:
resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/htmlparser2/-/htmlparser2-8.0.2.tgz}
name: htmlparser2
version: 8.0.2
dependencies:
'@tootallnate/once': registry.npmmirror.com/@tootallnate/once@2.0.0
agent-base: registry.npmmirror.com/agent-base@6.0.2
debug: registry.npmmirror.com/debug@4.3.4
transitivePeerDependencies:
- supports-color
domelementtype: registry.npmmirror.com/domelementtype@2.3.0
domhandler: registry.npmmirror.com/domhandler@5.0.3
domutils: registry.npmmirror.com/domutils@3.1.0
entities: registry.npmmirror.com/entities@4.5.0
dev: false
registry.npmmirror.com/https-browserify@1.0.0:
@ -8356,18 +8342,6 @@ packages:
version: 1.0.0
dev: true
registry.npmmirror.com/https-proxy-agent@5.0.1:
resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz}
name: https-proxy-agent
version: 5.0.1
engines: {node: '>= 6'}
dependencies:
agent-base: registry.npmmirror.com/agent-base@6.0.2
debug: registry.npmmirror.com/debug@4.3.4
transitivePeerDependencies:
- supports-color
dev: false
registry.npmmirror.com/human-signals@4.3.1:
resolution: {integrity: sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/human-signals/-/human-signals-4.3.1.tgz}
name: human-signals
@ -8743,12 +8717,6 @@ packages:
engines: {node: '>=12'}
dev: false
registry.npmmirror.com/is-potential-custom-element-name@1.0.1:
resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz}
name: is-potential-custom-element-name
version: 1.0.1
dev: false
registry.npmmirror.com/is-regex@1.1.4:
resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-regex/-/is-regex-1.1.4.tgz}
name: is-regex
@ -8921,46 +8889,6 @@ packages:
engines: {node: '>=0.1.90'}
dev: false
registry.npmmirror.com/jsdom@22.1.0:
resolution: {integrity: sha512-/9AVW7xNbsBv6GfWho4TTNjEo9fe6Zhf9O7s0Fhhr3u+awPwAJMKwAMXnkk5vBxflqLW9hTHX/0cs+P3gW+cQw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jsdom/-/jsdom-22.1.0.tgz}
name: jsdom
version: 22.1.0
engines: {node: '>=16'}
peerDependencies:
canvas: ^2.5.0
peerDependenciesMeta:
canvas:
optional: true
dependencies:
abab: registry.npmmirror.com/abab@2.0.6
cssstyle: registry.npmmirror.com/cssstyle@3.0.0
data-urls: registry.npmmirror.com/data-urls@4.0.0
decimal.js: registry.npmmirror.com/decimal.js@10.4.3
domexception: registry.npmmirror.com/domexception@4.0.0
form-data: registry.npmmirror.com/form-data@4.0.0
html-encoding-sniffer: registry.npmmirror.com/html-encoding-sniffer@3.0.0
http-proxy-agent: registry.npmmirror.com/http-proxy-agent@5.0.0
https-proxy-agent: registry.npmmirror.com/https-proxy-agent@5.0.1
is-potential-custom-element-name: registry.npmmirror.com/is-potential-custom-element-name@1.0.1
nwsapi: registry.npmmirror.com/nwsapi@2.2.7
parse5: registry.npmmirror.com/parse5@7.1.2
rrweb-cssom: registry.npmmirror.com/rrweb-cssom@0.6.0
saxes: registry.npmmirror.com/saxes@6.0.0
symbol-tree: registry.npmmirror.com/symbol-tree@3.2.4
tough-cookie: registry.npmmirror.com/tough-cookie@4.1.3
w3c-xmlserializer: registry.npmmirror.com/w3c-xmlserializer@4.0.0
webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
whatwg-encoding: registry.npmmirror.com/whatwg-encoding@2.0.0
whatwg-mimetype: registry.npmmirror.com/whatwg-mimetype@3.0.0
whatwg-url: registry.npmmirror.com/whatwg-url@12.0.1
ws: registry.npmmirror.com/ws@8.14.2
xml-name-validator: registry.npmmirror.com/xml-name-validator@4.0.0
transitivePeerDependencies:
- bufferutil
- supports-color
- utf-8-validate
dev: false
registry.npmmirror.com/jsesc@0.5.0:
resolution: {integrity: sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jsesc/-/jsesc-0.5.0.tgz}
name: jsesc
@ -10343,6 +10271,24 @@ packages:
whatwg-url: registry.npmmirror.com/whatwg-url@5.0.0
dev: false
registry.npmmirror.com/node-html-markdown@1.3.0:
resolution: {integrity: sha512-OeFi3QwC/cPjvVKZ114tzzu+YoR+v9UXW5RwSXGUqGb0qCl0DvP406tzdL7SFn8pZrMyzXoisfG2zcuF9+zw4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-html-markdown/-/node-html-markdown-1.3.0.tgz}
name: node-html-markdown
version: 1.3.0
engines: {node: '>=10.0.0'}
dependencies:
node-html-parser: registry.npmmirror.com/node-html-parser@6.1.11
dev: false
registry.npmmirror.com/node-html-parser@6.1.11:
resolution: {integrity: sha512-FAgwwZ6h0DSDWxfD0Iq1tsDcBCxdJB1nXpLPPxX8YyVWzbfCjKWEzaynF4gZZ/8hziUmp7ZSaKylcn0iKhufUQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-html-parser/-/node-html-parser-6.1.11.tgz}
name: node-html-parser
version: 6.1.11
dependencies:
css-select: registry.npmmirror.com/css-select@5.1.0
he: registry.npmmirror.com/he@1.2.0
dev: false
registry.npmmirror.com/node-releases@2.0.13:
resolution: {integrity: sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-releases/-/node-releases-2.0.13.tgz}
name: node-releases
@ -10416,13 +10362,6 @@ packages:
version: 2.1.1
dependencies:
boolbase: registry.npmmirror.com/boolbase@1.0.0
dev: true
registry.npmmirror.com/nwsapi@2.2.7:
resolution: {integrity: sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/nwsapi/-/nwsapi-2.2.7.tgz}
name: nwsapi
version: 2.2.7
dev: false
registry.npmmirror.com/object-assign@4.1.1:
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/object-assign/-/object-assign-4.1.1.tgz}
@ -10703,12 +10642,22 @@ packages:
json-parse-even-better-errors: registry.npmmirror.com/json-parse-even-better-errors@2.3.1
lines-and-columns: registry.npmmirror.com/lines-and-columns@1.2.4
registry.npmmirror.com/parse5-htmlparser2-tree-adapter@7.0.0:
resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz}
name: parse5-htmlparser2-tree-adapter
version: 7.0.0
dependencies:
domhandler: registry.npmmirror.com/domhandler@5.0.3
parse5: registry.npmmirror.com/parse5@7.1.2
dev: false
registry.npmmirror.com/parse5@7.1.2:
resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/parse5/-/parse5-7.1.2.tgz}
name: parse5
version: 7.1.2
dependencies:
entities: registry.npmmirror.com/entities@4.5.0
dev: false
registry.npmmirror.com/path-browserify@1.0.1:
resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/path-browserify/-/path-browserify-1.0.1.tgz}
@ -11060,12 +11009,6 @@ packages:
version: 1.1.0
dev: false
registry.npmmirror.com/psl@1.9.0:
resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/psl/-/psl-1.9.0.tgz}
name: psl
version: 1.9.0
dev: false
registry.npmmirror.com/public-encrypt@4.0.3:
resolution: {integrity: sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/public-encrypt/-/public-encrypt-4.0.3.tgz}
name: public-encrypt
@ -11106,12 +11049,6 @@ packages:
engines: {node: '>=0.4.x'}
dev: true
registry.npmmirror.com/querystringify@2.2.0:
resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/querystringify/-/querystringify-2.2.0.tgz}
name: querystringify
version: 2.2.0
dev: false
registry.npmmirror.com/queue-microtask@1.2.3:
resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/queue-microtask/-/queue-microtask-1.2.3.tgz}
name: queue-microtask
@ -11628,12 +11565,6 @@ packages:
engines: {node: '>=4'}
dev: false
registry.npmmirror.com/requires-port@1.0.0:
resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/requires-port/-/requires-port-1.0.0.tgz}
name: requires-port
version: 1.0.0
dev: false
registry.npmmirror.com/resolve-from@4.0.0:
resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/resolve-from/-/resolve-from-4.0.0.tgz}
name: resolve-from
@ -11724,12 +11655,6 @@ packages:
fsevents: registry.npmmirror.com/fsevents@2.3.3
dev: true
registry.npmmirror.com/rrweb-cssom@0.6.0:
resolution: {integrity: sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz}
name: rrweb-cssom
version: 0.6.0
dev: false
registry.npmmirror.com/run-parallel@1.2.0:
resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/run-parallel/-/run-parallel-1.2.0.tgz}
name: run-parallel
@ -11820,15 +11745,6 @@ packages:
immutable: registry.npmmirror.com/immutable@4.3.4
source-map-js: registry.npmmirror.com/source-map-js@1.0.2
registry.npmmirror.com/saxes@6.0.0:
resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/saxes/-/saxes-6.0.0.tgz}
name: saxes
version: 6.0.0
engines: {node: '>=v12.22.7'}
dependencies:
xmlchars: registry.npmmirror.com/xmlchars@2.2.0
dev: false
registry.npmmirror.com/scheduler@0.23.0:
resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/scheduler/-/scheduler-0.23.0.tgz}
name: scheduler
@ -12271,12 +12187,6 @@ packages:
stable: registry.npmmirror.com/stable@0.1.8
dev: true
registry.npmmirror.com/symbol-tree@3.2.4:
resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/symbol-tree/-/symbol-tree-3.2.4.tgz}
name: symbol-tree
version: 3.2.4
dev: false
registry.npmmirror.com/tapable@2.2.1:
resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tapable/-/tapable-2.2.1.tgz}
name: tapable
@ -12351,18 +12261,6 @@ packages:
version: 1.0.6
dev: false
registry.npmmirror.com/tough-cookie@4.1.3:
resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tough-cookie/-/tough-cookie-4.1.3.tgz}
name: tough-cookie
version: 4.1.3
engines: {node: '>=6'}
dependencies:
psl: registry.npmmirror.com/psl@1.9.0
punycode: registry.npmmirror.com/punycode@2.3.0
universalify: registry.npmmirror.com/universalify@0.2.0
url-parse: registry.npmmirror.com/url-parse@1.5.10
dev: false
registry.npmmirror.com/tr46@0.0.3:
resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tr46/-/tr46-0.0.3.tgz}
name: tr46
@ -12378,15 +12276,6 @@ packages:
punycode: registry.npmmirror.com/punycode@2.3.0
dev: false
registry.npmmirror.com/tr46@4.1.1:
resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tr46/-/tr46-4.1.1.tgz}
name: tr46
version: 4.1.1
engines: {node: '>=14'}
dependencies:
punycode: registry.npmmirror.com/punycode@2.3.0
dev: false
registry.npmmirror.com/trim-lines@3.0.1:
resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/trim-lines/-/trim-lines-3.0.1.tgz}
name: trim-lines
@ -12786,13 +12675,6 @@ packages:
unist-util-visit-parents: registry.npmmirror.com/unist-util-visit-parents@5.1.3
dev: false
registry.npmmirror.com/universalify@0.2.0:
resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/universalify/-/universalify-0.2.0.tgz}
name: universalify
version: 0.2.0
engines: {node: '>= 4.0.0'}
dev: false
registry.npmmirror.com/update-browserslist-db@1.0.13(browserslist@4.22.1):
resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz}
id: registry.npmmirror.com/update-browserslist-db/1.0.13
@ -12814,15 +12696,6 @@ packages:
punycode: registry.npmmirror.com/punycode@2.3.0
dev: true
registry.npmmirror.com/url-parse@1.5.10:
resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/url-parse/-/url-parse-1.5.10.tgz}
name: url-parse
version: 1.5.10
dependencies:
querystringify: registry.npmmirror.com/querystringify@2.2.0
requires-port: registry.npmmirror.com/requires-port@1.0.0
dev: false
registry.npmmirror.com/url@0.11.3:
resolution: {integrity: sha512-6hxOLGfZASQK/cijlZnZJTq8OXAkt/3YGfQX45vvMYXpZoo8NdWZcY73K108Jf759lS1Bv/8wXnHDTSz17dSRw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/url/-/url-0.11.3.tgz}
name: url
@ -13073,15 +12946,6 @@ packages:
version: 3.1.0
engines: {node: '>=0.10.0'}
registry.npmmirror.com/w3c-xmlserializer@4.0.0:
resolution: {integrity: sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz}
name: w3c-xmlserializer
version: 4.0.0
engines: {node: '>=14'}
dependencies:
xml-name-validator: registry.npmmirror.com/xml-name-validator@4.0.0
dev: false
registry.npmmirror.com/watchpack@2.4.0:
resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/watchpack/-/watchpack-2.4.0.tgz}
name: watchpack
@ -13130,22 +12994,6 @@ packages:
engines: {node: '>=12'}
dev: false
registry.npmmirror.com/whatwg-encoding@2.0.0:
resolution: {integrity: sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz}
name: whatwg-encoding
version: 2.0.0
engines: {node: '>=12'}
dependencies:
iconv-lite: registry.npmmirror.com/iconv-lite@0.6.3
dev: false
registry.npmmirror.com/whatwg-mimetype@3.0.0:
resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz}
name: whatwg-mimetype
version: 3.0.0
engines: {node: '>=12'}
dev: false
registry.npmmirror.com/whatwg-url@11.0.0:
resolution: {integrity: sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-11.0.0.tgz}
name: whatwg-url
@ -13156,16 +13004,6 @@ packages:
webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
dev: false
registry.npmmirror.com/whatwg-url@12.0.1:
resolution: {integrity: sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-12.0.1.tgz}
name: whatwg-url
version: 12.0.1
engines: {node: '>=14'}
dependencies:
tr46: registry.npmmirror.com/tr46@4.1.1
webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
dev: false
registry.npmmirror.com/whatwg-url@5.0.0:
resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-5.0.0.tgz}
name: whatwg-url
@ -13308,28 +13146,6 @@ packages:
name: wrappy
version: 1.0.2
registry.npmmirror.com/ws@8.14.2:
resolution: {integrity: sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/ws/-/ws-8.14.2.tgz}
name: ws
version: 8.14.2
engines: {node: '>=10.0.0'}
peerDependencies:
bufferutil: ^4.0.1
utf-8-validate: '>=5.0.2'
peerDependenciesMeta:
bufferutil:
optional: true
utf-8-validate:
optional: true
dev: false
registry.npmmirror.com/xml-name-validator@4.0.0:
resolution: {integrity: sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz}
name: xml-name-validator
version: 4.0.0
engines: {node: '>=12'}
dev: false
registry.npmmirror.com/xmlbuilder@10.1.1:
resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz}
name: xmlbuilder
@ -13337,12 +13153,6 @@ packages:
engines: {node: '>=4.0'}
dev: false
registry.npmmirror.com/xmlchars@2.2.0:
resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlchars/-/xmlchars-2.2.0.tgz}
name: xmlchars
version: 2.2.0
dev: false
registry.npmmirror.com/xtend@4.0.2:
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz}
name: xtend

3
projects/app/package.json
View File

@ -19,7 +19,6 @@
"@fastgpt/global": "workspace:*",
"@fastgpt/service": "workspace:*",
"@fastgpt/web": "workspace:*",
"@mozilla/readability": "^0.4.4",
"@node-rs/jieba": "^1.7.2",
"@tanstack/react-query": "^4.24.10",
"@types/nprogress": "^0.2.0",
@ -35,7 +34,6 @@
"i18next": "^22.5.1",
"immer": "^9.0.19",
"jschardet": "^3.0.0",
"jsdom": "^22.1.0",
"jsonwebtoken": "^9.0.2",
"lodash": "^4.17.21",
"mammoth": "^1.6.0",
@ -67,7 +65,6 @@
"@types/downloadjs": "^1.4.3",
"@types/formidable": "^2.0.5",
"@types/js-cookie": "^3.0.3",
"@types/jsdom": "^21.1.1",
"@types/jsonwebtoken": "^9.0.3",
"@types/lodash": "^4.14.191",
"@types/multer": "^1.4.10",

20
projects/app/public/docs/versionIntro.md
View File

@ -1,13 +1,11 @@
### Fast GPT V4.6.2
1. 新增 - 团队空间
2. 新增 - 多路向量（多个向量映射一组数据）
3. 新增 - tts语音
4. 新增 - 语音输入
5. 新增 - 增强召回方式，提高召回精度
6. 优化 - 知识库导出，可直接触发流下载，无需等待转圈圈
7. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
8. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词)
9. [使用文档](https://doc.fastgpt.in/docs/intro/)
10. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
11. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)
1. 商业版新增 - web站点同步
2. 新增 - 集合元数据记录
3. 优化 - url 读取内容
4. 优化 - 流读取文件，防止内存溢出
5. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
6. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词)
7. [使用文档](https://doc.fastgpt.in/docs/intro/)
8. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
9. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)

74
projects/app/public/locales/en/common.json
View File

@ -3,7 +3,6 @@
"Cancel": "No",
"Confirm": "Yes",
"Create New": "Create",
"Dataset": "Dataset",
"Export": "Export",
"Folder": "Folder",
"Move": "Move",
@ -129,6 +128,7 @@
"Choose": "Choose",
"Close": "Close",
"Collect": "Collect",
"Config": "Config",
"Confirm": "Confirm",
"Confirm Create": "Create",
"Confirm Move": "Move here",
@ -138,7 +138,6 @@
"Course": "",
"Create Failed": "Create Failed",
"Create Success": "Create Success",
"Create Virtual File Failed": "Create Virtual File Failed",
"Custom Title": "Custom Title",
"Delete": "Delete",
"Delete Failed": "Delete Failed",
@ -152,6 +151,7 @@
"Filed is repeat": "Filed is repeated",
"Filed is repeated": "",
"Input": "Input",
"Intro": "Intro",
"Last Step": "Last",
"Loading": "Loading",
"Max credit": "Credit",
@ -190,6 +190,14 @@
"Update Time": "Update Time",
"Upload File Failed": "Upload File Failed",
"Username": "UserName",
"Website": "Website",
"choosable": "choosable",
"confirm": {
"Common Tip": "Operational Confirm"
},
"empty": {
"Common Tip": "No data"
},
"error": {
"unKnow": "There was an accident"
},
@ -207,11 +215,15 @@
"input": {
"Repeat Value": "Repeat Value"
},
"link": {
"UnValid": "UnValid Link"
},
"speech": {
"error tip": "Speech Failed"
}
},
"core": {
"Max Token": "MaxTokens",
"ai": {
"Model": "Model",
"Prompt": "Prompt"
@ -238,12 +250,13 @@
"Speech model": "Speech model",
"Speech speed": "Speed",
"Test Listen": "Test",
"Test Listen Text": "Hello, this is FastGPT, how can I help you?",
"Test Listen Text": "Hello, this is a voice test, if you can hear this sentence, it means that the voice playback function is normal",
"Web": "Browser (free)"
}
},
"chat": {
"Audio Speech Error": "Audio Speech Error",
"Quote Amount": "Dataset Quote:{{amount}}",
"Record": "Speech",
"Restart": "Restart",
"Select File": "Select file",
@ -252,23 +265,46 @@
"Speaking": "I'm listening...",
"Stop Speak": "Stop Speak",
"Type a message": "Input problem",
"quote": {
"Quote Tip": "Only the actual reference content is displayed here. If the data is updated, it will not be updated in real time",
"Read Quote": "Read Quote",
"Read Source": "Read Source"
},
"tts": {
"Stop Speech": "Stop"
}
},
"dataset": {
"All Dataset": "All Dataset",
"Choose Dataset": "Choose Dataset",
"Common Dataset": "Common Dataset",
"Common Dataset Desc": "Knowledge bases can be built by importing files, web links, or manual entry",
"Create dataset": "Create Dataset",
"Dataset": "Dataset",
"Dataset Type": "Dataset Type",
"Delete Website Tips": "Confirm to delete the website",
"Empty Dataset": "",
"Empty Dataset Tips": "There is no knowledge base yet, go create one!",
"Folder Dataset": "Folder",
"Go Dataset": "To Dataset",
"Intro Placeholder": "This dataset has not yet been introduced~",
"Quote Length": "Quote Length",
"Read Dataset": "Read Dataset",
"Search Top K": "Top K",
"Set Empty Result Tip": ",Response empty text",
"Set Website Config": "Configuring Website",
"Similarity": "Similarity",
"Sync Time": "Update Time",
"Virtual File": "Virtual File",
"Website Dataset": "Website Sync",
"Website Dataset Desc": "Web site synchronization allows you to build a knowledge base directly from a web link",
"collection": {
"Click top config website": "Config",
"Empty Tip": "The collection is empty",
"Website Create Success": "Created successfully, data is being synchronized",
"Website Empty Tip": "No associated website yet,",
"Website Link": "Website Link",
"Website Sync": "Website",
"metadata": {
"Chunk Size": "Chunk Size",
"Createtime": "Create Time",
@ -281,6 +317,10 @@
"source name": "Source Name",
"source size": "Source Size"
},
"status": {
"active": "Ready",
"syncing": "Syncing"
},
"training": {
"type chunk": "Chunk",
"type manual": "Manual",
@ -295,6 +335,10 @@
"file": "File",
"folder": "Folder",
"import": {
"Fetch Error": "Get link failed",
"Fetch Url": "Url",
"Fetch url placeholder": "Up to 10 links, one per line.",
"Fetch url tip": "Only static links can be read, please check the results",
"Ideal chunk length": "Ideal chunk length",
"Ideal chunk length Tips": "Segment by end symbol. We recommend that your document should be properly punctuated to ensure that each complete sentence length does not exceed this value \n Chinese document recommended 400~1000\n English document recommended 600~1200"
},
@ -316,6 +360,10 @@
},
"search mode": "Search Mode"
},
"status": {
"active": "Ready",
"syncing": "Syncing"
},
"test": {
"Test": "Start",
"Test Result": "Results",
@ -327,8 +375,18 @@
"test result tip": "The contents of the knowledge base are sorted according to their similarity to the test text, and you can adjust the corresponding text according to the test results. Note: The data in the test record may have been modified, clicking on a test data will show the latest data."
},
"training": {
"Website Sync": "Website Sync",
"type chunk": "Chunk",
"type qa": "QA"
},
"website": {
"Base Url": "BaseUrl",
"Config": "Website Configuring",
"Config Description": "The Web site synchronization function allows you to fill in the root address of a website, and the system will automatically crawl the relevant pages deeply for knowledge base training. Only crawls static websites, mainly project documents and blogs.",
"Confirm Create Tips": "Confirm to synchronize the site, the synchronization task will start later, please confirm!",
"Confirm Update Tips": "Are you sure to update the site configuration? The synchronization starts immediately with the new configuration. Please confirm",
"Selector": "Selector",
"Start Sync": "Start Sync"
}
},
"module": {
@ -434,7 +492,6 @@
"Create Training Data": "Training-{{filename}}",
"Create Virtual File Success": "Create Virtual File Success",
"Data Amount": "Data Amount",
"Ready": "Ready",
"Select Collection": "Select Collection",
"Select One Collection To Store": "Select the collection to store"
},
@ -466,6 +523,10 @@
"import csv tip": "Ensure that the CSV is in UTF-8 format; otherwise, garbled characters will be displayed",
"test": {
"noResult": "Search results are empty"
},
"website": {
"Base Url": "BaseUrl",
"Selector": "Selector"
}
},
"error": {
@ -482,7 +543,6 @@
"Fetch Url": "Fetch Url",
"If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format",
"Parse": "{{name}} Parsing...",
"Ready": "Ready",
"Release the mouse to upload the file": "Release the mouse to upload the file",
"Select a maximum of 10 files": "Select a maximum of 10 files",
"Uploading": "Uploading: {{name}}, Progress: {{percent}}%",
@ -725,6 +785,10 @@
"ReRank": "ReRank",
"Whisper": "Whisper",
"bill username": "User"
},
"moduleName": {
"index": "Index Generation",
"qa": "QA Generation"
}
}
}

76
projects/app/public/locales/zh/common.json
View File

@ -3,7 +3,6 @@
"Cancel": "取消",
"Confirm": "确认",
"Create New": "新建",
"Dataset": "知识库",
"Export": "导出",
"Folder": "文件夹",
"Move": "移动",
@ -129,6 +128,7 @@
"Choose": "选择",
"Close": "关闭",
"Collect": "收藏",
"Config": "配置",
"Confirm": "确认",
"Confirm Create": "确认创建",
"Confirm Move": "移动到这",
@ -138,7 +138,6 @@
"Course": "",
"Create Failed": "创建异常",
"Create Success": "创建成功",
"Create Virtual File Failed": "创建虚拟文件失败",
"Custom Title": "自定义标题",
"Delete": "删除",
"Delete Failed": "删除失败",
@ -152,6 +151,7 @@
"Filed is repeat": "",
"Filed is repeated": "字段重复了",
"Input": "输入",
"Intro": "介绍",
"Last Step": "上一步",
"Loading": "加载中",
"Max credit": "最大金额",
@ -190,6 +190,14 @@
"Update Time": "更新时间",
"Upload File Failed": "上传文件失败",
"Username": "用户名",
"Website": "网站",
"choosable": "可选",
"confirm": {
"Common Tip": "操作确认"
},
"empty": {
"Common Tip": "没有什么数据噢~"
},
"error": {
"unKnow": "出现了点意外~"
},
@ -207,11 +215,15 @@
"input": {
"Repeat Value": "有重复的值"
},
"link": {
"UnValid": "无效的链接"
},
"speech": {
"error tip": "语音转文字失败"
}
},
"core": {
"Max Token": "单条数据上限",
"ai": {
"Model": "AI 模型",
"Prompt": "提示词"
@ -238,12 +250,13 @@
"Speech model": "语音模型",
"Speech speed": "语速",
"Test Listen": "试听",
"Test Listen Text": "你好，我是 FastGPT，有什么可以帮助你么？",
"Test Listen Text": "你好，这是语音测试，如果你能听到这句话，说明语音播放功能正常",
"Web": "浏览器自带(免费)"
}
},
"chat": {
"Audio Speech Error": "语音播报异常",
"Quote Amount": "知识库引用({{amount}}条)",
"Record": "语音输入",
"Restart": "重开对话",
"Select File": "选择文件",
@ -252,23 +265,46 @@
"Speaking": "我在听，请说...",
"Stop Speak": "停止录音",
"Type a message": "输入问题",
"quote": {
"Quote Tip": "此处仅显示实际引用内容，若数据有更新，此处不会实时更新",
"Read Quote": "查看引用",
"Read Source": "查看来源"
},
"tts": {
"Stop Speech": "停止"
}
},
"dataset": {
"All Dataset": "全部知识库",
"Choose Dataset": "关联知识库",
"Common Dataset": "通用知识库",
"Common Dataset Desc": "可通过导入文件、网页链接或手动录入形式构建知识库",
"Create dataset": "创建一个知识库",
"Dataset": "知识库",
"Dataset Type": "知识库类型",
"Delete Website Tips": "确认删除该站点？",
"Empty Dataset": "",
"Empty Dataset Tips": "还没有知识库，快去创建一个吧！",
"Folder Dataset": "文件夹",
"Go Dataset": "前往知识库",
"Intro Placeholder": "这个知识库还没有介绍~",
"Quote Length": "引用内容长度",
"Read Dataset": "查看知识库详情",
"Search Top K": "单次搜索数量",
"Set Empty Result Tip": ",未搜索到内容时回复指定内容",
"Set Website Config": "开始配置网站信息",
"Similarity": "相似度",
"Sync Time": "最后更新时间",
"Virtual File": "虚拟文件",
"Website Dataset": "Web 站点同步",
"Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库",
"collection": {
"Click top config website": "点击配置网站",
"Empty Tip": "数据集空空如也",
"Website Create Success": "创建成功，正在同步数据",
"Website Empty Tip": "还没有关联网站，",
"Website Link": "Web 站点地址",
"Website Sync": "Web 站点同步",
"metadata": {
"Chunk Size": "分割大小",
"Createtime": "创建时间",
@ -281,9 +317,13 @@
"source name": "来源名",
"source size": "来源大小"
},
"status": {
"active": "已就绪",
"syncing": "同步中"
},
"training": {
"type manual": "手动",
"type chunk": "直接分段",
"type manual": "手动",
"type qa": "问答拆分"
}
},
@ -295,6 +335,10 @@
"file": "文件",
"folder": "目录",
"import": {
"Fetch Error": "获取链接失败",
"Fetch Url": "网络链接",
"Fetch url placeholder": "最多10个链接，每行一个。",
"Fetch url tip": "仅支持读取静态链接，请注意检查结果",
"Ideal chunk length": "理想分块长度",
"Ideal chunk length Tips": "按结束符号进行分段。我们建议您的文档应合理的使用标点符号，以确保每个完整的句子长度不要超过该值\n中文文档建议400~1000\n英文文档建议600~1200"
},
@ -316,6 +360,10 @@
},
"search mode": "检索模式"
},
"status": {
"active": "已就绪",
"syncing": "同步中"
},
"test": {
"Test": "测试",
"Test Result": "测试结果",
@ -327,8 +375,18 @@
"test result tip": "根据知识库内容与测试文本的相似度进行排序，你可以根据测试结果调整对应的文本。\n注意：测试记录中的数据可能已经被修改过，点击某条测试数据后将展示最新的数据。"
},
"training": {
"Website Sync": "Web 站点同步",
"type chunk": "直接分段",
"type qa": "问答拆分"
},
"website": {
"Base Url": "根地址",
"Config": "Web站点配置",
"Config Description": "Web 站点同步功能允许你填写一个网站的根地址，系统会自动深度抓取相关的网页进行知识库训练。仅会抓取静态的网站，以项目文档、博客为主。",
"Confirm Create Tips": "确认同步该站点，同步任务将随后开启，请确认！",
"Confirm Update Tips": "确认更新站点配置？会立即按新的配置开始同步，请确认！",
"Selector": "选择器",
"Start Sync": "开始同步"
}
},
"module": {
@ -434,7 +492,6 @@
"Create Training Data": "文件训练-{{filename}}",
"Create Virtual File Success": "创建虚拟文件成功",
"Data Amount": "数据总量",
"Ready": "已就绪",
"Select Collection": "选择文件",
"Select One Collection To Store": "选择一个文件进行存储"
},
@ -466,6 +523,10 @@
"import csv tip": "请确保CSV为UTF-8格式，否则会乱码",
"test": {
"noResult": "搜索结果为空"
},
"website": {
"Base Url": "",
"Selector": ""
}
},
"error": {
@ -482,7 +543,6 @@
"Fetch Url": "链接读取",
"If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式",
"Parse": "{{name}} 解析中...",
"Ready": "",
"Release the mouse to upload the file": "松开鼠标上传文件",
"Select a maximum of 10 files": "最多选择10个文件",
"Uploading": "正在上传 {{name}}，进度: {{percent}}%",
@ -725,6 +785,10 @@
"ReRank": "结果重排",
"Whisper": "语音输入",
"bill username": "用户"
},
"moduleName": {
"index": "索引生成",
"qa": "QA 拆分"
}
}
}

4
projects/app/src/components/ChatBox/MessageInput.tsx
View File

@ -74,8 +74,8 @@ const MessageInput = ({
try {
const src = await compressImgFileAndUpload({
file: file.rawFile,
maxW: 1000,
maxH: 1000,
maxW: 4329,
maxH: 4329,
maxSize: 1024 * 1024 * 5,
// 30 day expired.
expiredTime: addDays(new Date(), 30)

6
projects/app/src/components/ChatBox/QuoteModal.tsx
View File

@ -71,9 +71,9 @@ const QuoteModal = ({
iconSrc="/imgs/modal/quote.svg"
title={
<Box>
知识库引用({rawSearch.length}条)
<Box fontSize={'10px'} color={'myGray.500'} fontWeight={'normal'}>
注意: 修改知识库内容成功后，此处不会显示变更情况。点击编辑后，会显示知识库最新的内容。
{t('core.chat.Quote Amount', { amount: rawSearch.length })}
<Box fontSize={'sm'} color={'myGray.500'} fontWeight={'normal'}>
{t('core.chat.quote.Quote Tip')}
</Box>
</Box>
}

60
projects/app/src/components/ChatBox/ResponseTags.tsx
View File

@ -1,7 +1,7 @@
import React, { useMemo, useState } from 'react';
import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/api.d';
import type { ChatItemType } from '@fastgpt/global/core/chat/type';
import { Flex, BoxProps, useDisclosure, Image, useTheme } from '@chakra-ui/react';
import { Flex, BoxProps, useDisclosure, Image, useTheme, Box } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
@ -11,6 +11,8 @@ import MyTooltip from '../MyTooltip';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
import ChatBoxDivider from '@/components/core/chat/Divider';
import MyIcon from '../Icon';
import { getFileAndOpen } from '@/web/core/dataset/utils';
const QuoteModal = dynamic(() => import('./QuoteModal'), { ssr: false });
const ContextModal = dynamic(() => import('./ContextModal'), { ssr: false });
@ -59,6 +61,7 @@ const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemRes
.flat()
.map((item) => ({
sourceName: item.sourceName,
sourceId: item.sourceId,
icon: getSourceNameIcon({ sourceId: item.sourceId, sourceName: item.sourceName })
})),
historyPreview: chatData?.historyPreview,
@ -83,18 +86,67 @@ const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemRes
alignItems={'center'}
flexWrap={'wrap'}
fontSize={'sm'}
cursor={'pointer'}
border={theme.borders.sm}
py={1}
px={2}
borderRadius={'md'}
_hover={{
bg: 'myBlue.100'
'.controller': {
display: 'flex'
}
}}
overflow={'hidden'}
position={'relative'}
onClick={() => setQuoteModalData(quoteList)}
>
<Image src={item.icon} alt={''} mr={1} w={'12px'} />
{item.sourceName}
<Box className="textEllipsis" flex={'1 0 0'}>
{item.sourceName}
</Box>
<Box
className="controller"
display={['flex', 'none']}
pr={2}
position={'absolute'}
right={0}
left={0}
justifyContent={'flex-end'}
alignItems={'center'}
h={'100%'}
lineHeight={0}
bg={`linear-gradient(to left, white,white ${
item.sourceId ? '60px' : '30px'
}, rgba(255,255,255,0) 80%)`}
>
<MyTooltip label={t('core.chat.quote.Read Quote')}>
<MyIcon
name="common/viewLight"
w={'14px'}
cursor={'pointer'}
_hover={{
color: 'green.600'
}}
/>
</MyTooltip>
{item.sourceId && (
<MyTooltip label={t('core.chat.quote.Read Source')}>
<MyIcon
ml={4}
name="common/routePushLight"
w={'14px'}
cursor={'pointer'}
_hover={{ color: 'myBlue.600' }}
onClick={async (e) => {
e.stopPropagation();
if (!item.sourceId) return;
await getFileAndOpen(item.sourceId);
}}
/>
</MyTooltip>
)}
</Box>
</Flex>
))}
</Flex>

8
projects/app/src/components/ChatBox/index.module.scss
View File

@ -1,14 +1,6 @@
.stopIcon {
animation: zoomStopIcon 0.4s infinite alternate;
}
@keyframes zoomStopIcon {
0% {
transform: scale(0.8);
}
100% {
transform: scale(1.2);
}
}
.statusAnimation {
animation: statusBox 0.8s linear infinite alternate;

6
projects/app/src/components/EmptyTip/index.tsx
View File

@ -1,17 +1,19 @@
import React from 'react';
import { Flex, Box, FlexProps } from '@chakra-ui/react';
import MyIcon from '../Icon';
import { useTranslation } from 'next-i18next';
type Props = FlexProps & {
text?: string | null;
text?: string | React.ReactNode;
};
const EmptyTip = ({ text, ...props }: Props) => {
const { t } = useTranslation();
return (
<Flex mt={5} flexDirection={'column'} alignItems={'center'} pt={'10vh'} {...props}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
{text || '没有什么数据噢~'}
{text || t('common.empty.Common Tip')}
</Box>
</Flex>
);

1
projects/app/src/components/Icon/icons/common/confirm/commonTip.svg Normal file
View File

@ -0,0 +1 @@
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701410305116" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="8647" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M138.9056 180.5312m94.7712 0l563.3536 0q94.7712 0 94.7712 94.7712l0 576q0 94.7712-94.7712 94.7712l-563.3536 0q-94.7712 0-94.7712-94.7712l0-576q0-94.7712 94.7712-94.7712Z" fill="#CCDAFF" p-id="8648"></path><path d="M292.3008 81.92m64.768 0l305.3056 0q64.768 0 64.768 64.768l0 70.4q0 64.768-64.768 64.768l-305.3056 0q-64.768 0-64.768-64.768l0-70.4q0-64.768 64.768-64.768Z" fill="#7A7AF9" p-id="8649"></path><path d="M480.768 736.8704a49.408 49.408 0 0 1-34.7648-14.2848L332.8 611.3792a49.6128 49.6128 0 1 1 69.7344-70.7072l77.7216 76.4928L634.5216 460.8a49.6128 49.6128 0 1 1 70.6048 69.6832L516.096 721.92a49.6128 49.6128 0 0 1-35.328 14.9504z" fill="#7A7AF9" p-id="8650"></path></svg>
Side by Side

After

Width:  |  Height:  |  Size: 1017 B

1
projects/app/src/components/Icon/icons/common/confirm/deleteTip.svg Normal file
View File

@ -0,0 +1 @@
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701403554068" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5098" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M572.074667 337.134933a57.275733 57.275733 0 1 0-114.176-6.007466h-0.170667l12.936533 272.896v0.443733a44.544 44.544 0 1 0 89.019734-1.194667l12.424533-266.1376z m-196.949334-191.214933c76.151467-130.048 199.816533-129.774933 275.797334 0l340.3776 581.085867c76.117333 130.048 15.7696 235.4176-135.236267 235.4176H169.984c-150.8352 0-211.217067-105.6768-135.202133-235.4176L375.125333 145.92z m140.049067 687.581867a57.275733 57.275733 0 1 0 0-114.517334 57.275733 57.275733 0 0 0 0 114.517334z" fill="#FB6547" p-id="5099"></path></svg>
Side by Side

After

Width:  |  Height:  |  Size: 869 B

11
projects/app/src/components/Icon/icons/common/routePushLight.svg Normal file
View File

@ -0,0 +1,11 @@
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701418384907"
class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="11367"
xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128">
<path
d="M892 928.1H134c-19.9 0-36-16.1-36-36v-758c0-19.9 16.1-36 36-36h314.1c19.9 0 36 16.1 36 36s-16.1 36-36 36H170v686h686V579.6c0-19.9 16.1-36 36-36s36 16.1 36 36v312.5c0 19.9-16.1 36-36 36z"
p-id="11368"></path>
<path
d="M927.9 131.6v-0.5c-0.1-1.7-0.4-3.3-0.7-4.9 0-0.1 0-0.2-0.1-0.3-0.4-1.7-0.9-3.3-1.5-4.9v-0.1c-0.6-1.6-1.4-3.1-2.2-4.6 0-0.1-0.1-0.1-0.1-0.2-0.8-1.4-1.7-2.8-2.7-4.1-0.1-0.1-0.2-0.3-0.3-0.4-0.5-0.6-0.9-1.1-1.4-1.7 0-0.1-0.1-0.1-0.1-0.2-0.5-0.6-1-1.1-1.6-1.6l-0.4-0.4c-0.5-0.5-1.1-1-1.6-1.5l-0.1-0.1c-0.6-0.5-1.2-1-1.9-1.4-0.1-0.1-0.3-0.2-0.4-0.3-1.4-1-2.8-1.8-4.3-2.6l-0.1-0.1c-1.6-0.8-3.2-1.5-4.9-2-1.6-0.5-3.3-1-5-1.2-0.1 0-0.2 0-0.3-0.1l-2.4-0.3h-0.3c-0.7-0.1-1.3-0.1-2-0.1H640.1c-19.9 0-36 16.1-36 36s16.1 36 36 36h165L487.6 487.6c-14.1 14.1-14.1 36.9 0 50.9 7 7 16.2 10.5 25.5 10.5 9.2 0 18.4-3.5 25.5-10.5L856 221v162.8c0 19.9 16.1 36 36 36s36-16.1 36-36V134.1c0-0.8 0-1.7-0.1-2.5z"
p-id="11369"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.3 KiB

1
projects/app/src/components/Icon/icons/common/viewLight.svg Normal file
View File

@ -0,0 +1 @@
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701420318127" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="13238" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M987.52 367.36c-108.16-146.56-266.88-239.36-444.16-239.36-177.28 0-335.36 92.8-444.16 239.36-48 64.64-48 160.64 0 225.28C208 739.2 366.72 832 543.36 832c177.28 0 335.36-92.8 444.16-239.36C1035.52 528 1035.52 432 987.52 367.36zM939.52 539.52C839.04 684.8 694.4 768 542.72 768c-151.68 0-296.32-83.2-396.8-228.48C121.6 505.6 121.6 455.04 145.92 420.48 246.4 275.2 391.04 192 542.72 192c151.68 0 296.32 83.2 396.8 228.48C963.2 455.04 963.2 505.6 939.52 539.52zM544 256C420.48 256 320 356.48 320 480S420.48 704 544 704 768 603.52 768 480 667.52 256 544 256zM544 640C455.68 640 384 568.32 384 480S455.68 320 544 320C632.32 320 704 391.68 704 480S632.32 640 544 640z" p-id="13239"></path></svg>
Side by Side

After

Width:  |  Height:  |  Size: 1021 B

1
projects/app/src/components/Icon/icons/core/dataset/commonDataset.svg Normal file
View File

@ -0,0 +1 @@
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1697622173220" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="2398" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M722.36246914 1024.8960632h-415.54678519c-120.26260543 0-218.12969876-97.86709333-218.12969877-218.12969875v-586.42583704c0-120.26260543 97.86709333-218.12969876 218.12969877-218.12969877h415.54678519c120.26260543 0 218.12969876 97.86709333 218.12969876 218.12969877v586.42583704c0 120.26260543-97.86709333 218.12969876-218.12969876 218.12969875z m-415.54678519-898.28010667c-51.65207703 0-93.72457086 42.07249383-93.72457086 93.72457088v586.42583704c0 51.65207703 42.07249383 93.72457086 93.72457086 93.72457086h415.54678519c51.65207703 0 93.72457086-42.07249383 93.72457086-93.72457086v-586.42583704c0-51.65207703-42.07249383-93.72457086-93.72457086-93.72457088h-415.54678519z" fill="#BDD2EF" p-id="2399"></path><path d="M684.0441363 599.76969482H338.27296395c-42.07249383 0-76.11885037-34.04635653-76.11885037-76.11885037s34.04635653-76.11885037 76.11885037-76.11885037h345.77117235c42.07249383 0 76.11885037 34.04635653 76.11885037 76.11885037s-34.17581037 76.11885037-76.11885037 76.11885037zM465.00826075 360.66847605h-126.7352968c-42.07249383 0-76.11885037-34.04635653-76.11885037-76.11885036s34.04635653-76.11885037 76.11885037-76.11885038h126.7352968c42.07249383 0 76.11885037 34.04635653 76.11885036 76.11885038s-34.04635653 76.11885037-76.11885036 76.11885036zM684.0441363 360.66847605h-25.37295013c-42.07249383 0-76.11885037-34.04635653-76.11885036-76.11885036s34.04635653-76.11885037 76.11885036-76.11885038h25.37295013c42.07249383 0 76.11885037 34.04635653 76.11885037 76.11885038s-34.17581037 76.11885037-76.11885037 76.11885036zM684.0441363 838.87091358H338.27296395c-42.07249383 0-76.11885037-34.04635653-76.11885037-76.11885038s34.04635653-76.11885037 76.11885037-76.11885037h345.77117235c42.07249383 0 76.11885037 34.04635653 76.11885037 76.11885037s-34.17581037 76.11885037-76.11885037 76.11885038z" fill="#2867CE" p-id="2400"></path></svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.1 KiB

1
projects/app/src/components/Icon/icons/core/dataset/folderDataset.svg Normal file
View File

@ -0,0 +1 @@
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694141197423" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4891" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M855.04 385.024q19.456 2.048 38.912 10.24t33.792 23.04 21.504 37.376 2.048 54.272q-2.048 8.192-8.192 40.448t-14.336 74.24-18.432 86.528-19.456 76.288q-5.12 18.432-14.848 37.888t-25.088 35.328-36.864 26.112-51.2 10.24l-567.296 0q-21.504 0-44.544-9.216t-42.496-26.112-31.744-40.96-12.288-53.76l0-439.296q0-62.464 33.792-97.792t95.232-35.328l503.808 0q22.528 0 46.592 8.704t43.52 24.064 31.744 35.84 12.288 44.032l0 11.264-53.248 0q-40.96 0-95.744-0.512t-116.736-0.512-115.712-0.512-92.672-0.512l-47.104 0q-26.624 0-41.472 16.896t-23.04 44.544q-8.192 29.696-18.432 62.976t-18.432 61.952q-10.24 33.792-20.48 65.536-2.048 8.192-2.048 13.312 0 17.408 11.776 29.184t29.184 11.776q31.744 0 43.008-39.936l54.272-198.656q133.12 1.024 243.712 1.024l286.72 0z" fill="#FFCC66" p-id="4892"></path></svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.1 KiB

14
projects/app/src/components/Icon/icons/core/dataset/websiteDataset.svg Normal file
View File

@ -0,0 +1,14 @@
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701324062325"
class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4651"
xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128">
<path d="M512 512m-512 0a512 512 0 1 0 1024 0 512 512 0 1 0-1024 0Z" fill="#c6dafe" p-id="4652"></path>
<path d="M486.4 665.6h51.2v153.6h-51.2z" fill="#515151" p-id="4653"
data-spm-anchor-id="a313x.search_index.0.i3.5ac83a811R1Mg6" class="selected"></path>
<path
d="M435.2 768h153.6v51.2h-153.6zM230.4 281.6v384h563.2V281.6H230.4z m0-51.2h563.2a51.2 51.2 0 0 1 51.2 51.2v384a51.2 51.2 0 0 1-51.2 51.2H230.4a51.2 51.2 0 0 1-51.2-51.2V281.6a51.2 51.2 0 0 1 51.2-51.2z"
fill="#515151" p-id="4654" data-spm-anchor-id="a313x.search_index.0.i1.5ac83a811R1Mg6" class="selected"></path>
<path
d="M662.272 388.864c6.144-21.5296 8.448-63.0272-29.952-68.3776-30.72-3.84-67.584 16.128-86.8352 29.2096-6.144-0.768-13.056-1.536-19.968-1.536-49.9712-0.7936-82.2016 16.128-109.8496 52.224-9.984 13.056-19.2 35.3536-22.2976 60.672 13.824-23.808 56.8576-67.584 102.1952-85.248 0 0-68.352 49.152-102.1952 118.3232v0.768c-1.5616 3.072-3.072 6.144-4.6336 9.984-33.024 82.2016-6.144 117.5552 19.2256 123.6992 23.04 6.144 55.3216-5.376 81.408-33.024 43.776 9.984 87.6288-1.536 103.7312-9.984 30.72-16.896 52.2496-46.848 56.8832-77.6192h-84.5312s-3.84 26.88-49.1776 26.88c-41.472 0-43.776-48.384-43.776-48.384h178.9952s3.072-52.2496-22.272-86.8352c-14.6176-19.2-33.8176-36.1216-60.6976-45.312 8.448-6.144 22.272-15.4112 34.56-18.4832 23.04-6.144 38.4256-2.304 48.4096 14.592 13.056 23.04-7.68 76.0832-7.68 76.0832s12.288-15.36 18.432-37.632z m-185.1648 203.6224c-35.328 29.2096-64.512 25.3696-76.0576 8.4736-9.984-15.36-11.52-42.2656 0-79.8976 5.376 14.592 13.824 28.416 26.112 40.704 15.36 15.36 32.256 25.344 49.9456 30.72z m-1.536-149.8368s1.536-34.56 39.168-37.632c32.3072-3.072 49.2032 11.52 54.5536 39.1936l-93.7472-1.536z"
fill="#2394FB" p-id="4655"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.1 KiB

9
projects/app/src/components/Icon/index.tsx
View File

@ -114,7 +114,14 @@ const iconPaths = {
'core/dataset/modeEmbFTRerank': () => import('./icons/core/dataset/modeEmbFTRerank.svg'),
'core/app/variable/input': () => import('./icons/core/app/variable/input.svg'),
'core/app/variable/textarea': () => import('./icons/core/app/variable/textarea.svg'),
'core/app/variable/select': () => import('./icons/core/app/variable/select.svg')
'core/app/variable/select': () => import('./icons/core/app/variable/select.svg'),
'core/dataset/websiteDataset': () => import('./icons/core/dataset/websiteDataset.svg'),
'core/dataset/commonDataset': () => import('./icons/core/dataset/commonDataset.svg'),
'core/dataset/folderDataset': () => import('./icons/core/dataset/folderDataset.svg'),
'common/confirm/deleteTip': () => import('./icons/common/confirm/deleteTip.svg'),
'common/confirm/commonTip': () => import('./icons/common/confirm/commonTip.svg'),
'common/routePushLight': () => import('./icons/common/routePushLight.svg'),
'common/viewLight': () => import('./icons/common/viewLight.svg')
};
export type IconName = keyof typeof iconPaths;

2
projects/app/src/components/Layout/index.tsx
View File

@ -111,7 +111,7 @@ const Layout = ({ children }: { children: JSX.Element }) => {
</>
)}
</Box>
<Loading loading={loading} zIndex={9999} />
<Loading loading={loading} zIndex={999999} />
{!!userInfo && <UpdateInviteModal />}
</>
);

25
projects/app/src/components/Markdown/chat/Image.tsx
View File

@ -1,30 +1,33 @@
import { Box, Flex } from '@chakra-ui/react';
import { Box, Flex, Grid } from '@chakra-ui/react';
import MdImage from '../img/Image';
import { useMemo } from 'react';
const ImageBlock = ({ images }: { images: string }) => {
const formatData = useMemo(
() =>
images.split('\n').map((item) => {
try {
return JSON.parse(item) as { src: string };
} catch (error) {
return { src: '' };
}
}),
images
.split('\n')
.filter((item) => item)
.map((item) => {
try {
return JSON.parse(item) as { src: string };
} catch (error) {
return { src: '' };
}
}),
[images]
);
return (
<Flex alignItems={'center'} wrap={'wrap'} gap={4}>
<Grid gridTemplateColumns={['1fr', '1fr 1fr']} gap={4}>
{formatData.map(({ src }) => {
return (
<Box key={src} rounded={'md'} flex={'0 0 auto'} w={'120px'}>
<Box key={src} rounded={'md'} flex={'1 0 0'} minW={'120px'}>
<MdImage src={src} />
</Box>
);
})}
</Flex>
</Grid>
);
};

1
projects/app/src/components/Markdown/img/Image.tsx
View File

@ -29,7 +29,6 @@ const MdImage = ({ src }: { src?: string }) => {
borderRadius={'md'}
src={src}
alt={''}
maxH={'150px'}
fallbackSrc={'/imgs/errImg.png'}
fallbackStrategy={'onError'}
cursor={succeed ? 'pointer' : 'default'}

11
projects/app/src/components/MyModal/index.tsx
View File

@ -9,6 +9,7 @@ import {
Box,
Image
} from '@chakra-ui/react';
import MyIcon from '../Icon';
export interface MyModalProps extends ModalContentProps {
iconSrc?: string;
@ -56,7 +57,15 @@ const MyModal = ({
roundedTop={'lg'}
py={'10px'}
>
{iconSrc && <Image mr={3} objectFit={'contain'} alt="" src={iconSrc} w={'20px'} />}
{iconSrc && (
<>
{iconSrc.startsWith('/') ? (
<Image mr={3} objectFit={'contain'} alt="" src={iconSrc} w={'20px'} />
) : (
<MyIcon mr={3} name={iconSrc as any} w={'20px'} />
)}
</>
)}
{title}
<Box flex={1} />
{onClose && <ModalCloseButton position={'relative'} top={0} right={0} />}

15
projects/app/src/components/common/MyRadio/index.tsx
View File

@ -1,5 +1,5 @@
import React from 'react';
import { Box, Flex, useTheme, Grid, type GridProps, theme } from '@chakra-ui/react';
import { Box, Flex, useTheme, Grid, type GridProps, theme, Image } from '@chakra-ui/react';
import MyIcon from '@/components/Icon';
import { useTranslation } from 'next-i18next';
@ -45,8 +45,7 @@ const MyRadio = ({
: {
bg: 'myWhite.300',
_hover: {
bg: '#f5f8ff',
borderColor: '#b2ccff'
borderColor: 'myBlue.500'
}
})}
_after={{
@ -71,7 +70,15 @@ const MyRadio = ({
}}
onClick={() => onChange(item.value)}
>
{!!item.icon && <MyIcon mr={'14px'} name={item.icon as any} w={iconSize} />}
{!!item.icon && (
<>
{item.icon.startsWith('/') ? (
<Image src={item.icon} mr={'14px'} w={iconSize} alt={''} />
) : (
<MyIcon mr={'14px'} name={item.icon as any} w={iconSize} />
)}
</>
)}
<Box pr={2}>
<Box>{t(item.title)}</Box>
{!!item.desc && (

21
projects/app/src/components/core/module/DatasetSelectModal.tsx
View File

@ -7,29 +7,20 @@ import {
ModalBody,
ModalFooter,
useTheme,
Textarea,
Grid,
Divider,
Switch,
Image
Divider
} from '@chakra-ui/react';
import Avatar from '@/components/Avatar';
import { useForm } from 'react-hook-form';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import type { SelectedDatasetType } from '@fastgpt/global/core/module/api.d';
import { useToast } from '@/web/common/hooks/useToast';
import MySlider from '@/components/Slider';
import MyTooltip from '@/components/MyTooltip';
import MyModal from '@/components/MyModal';
import MyIcon from '@/components/Icon';
import { DatasetSearchModeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { useTranslation } from 'next-i18next';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { feConfigs } from '@/web/common/system/staticData';
import DatasetSelectContainer, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
import { useLoading } from '@/web/common/hooks/useLoading';
import EmptyTip from '@/components/EmptyTip';
import { AppSimpleEditFormType } from '@fastgpt/global/core/app/type';
export const DatasetSelectModal = ({
isOpen,
@ -132,9 +123,9 @@ export const DatasetSelectModal = ({
<MyTooltip
key={item._id}
label={
item.type === DatasetTypeEnum.dataset
? t('dataset.Select Dataset')
: t('dataset.Select Folder')
item.type === DatasetTypeEnum.folder
? t('dataset.Select Folder')
: t('dataset.Select Dataset')
}
>
<Card
@ -149,7 +140,7 @@ export const DatasetSelectModal = ({
onClick={() => {
if (item.type === DatasetTypeEnum.folder) {
setParentId(item._id);
} else if (item.type === DatasetTypeEnum.dataset) {
} else {
const vectorModel = selectedDatasets[0]?.vectorModel?.model;
if (vectorModel && vectorModel !== item.vectorModel.model) {

4
projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx
View File

@ -51,8 +51,8 @@ function EditModal({
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
setRefresh((state) => !state);

6
projects/app/src/constants/dataset.ts
View File

@ -14,7 +14,8 @@ export const defaultDatasetDetail: DatasetItemType = {
type: 'dataset',
avatar: '/icon/logo.svg',
name: '',
tags: [],
intro: '',
status: 'active',
permission: 'private',
isOwner: false,
canWrite: false,
@ -36,7 +37,8 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
type: 'dataset',
avatar: '/icon/logo.svg',
name: '',
tags: [],
intro: '',
status: 'active',
permission: 'private',
vectorModel: defaultVectorModels[0].model,
agentModel: defaultQAModels[0].model

20
projects/app/src/global/core/api/datasetReq.d.ts vendored
View File

@ -12,15 +12,6 @@ import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'
import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
/* ===== dataset ===== */
export type DatasetUpdateParams = {
id: string;
parentId?: string;
tags?: string[];
name?: string;
avatar?: string;
permission?: `${PermissionTypeEnum}`;
agentModel?: LLMModelItemType;
};
/* ======= collections =========== */
export type GetDatasetCollectionsProps = RequestPaging & {
@ -30,16 +21,7 @@ export type GetDatasetCollectionsProps = RequestPaging & {
simple?: boolean;
selectFolder?: boolean;
};
export type CreateDatasetCollectionParams = {
datasetId: string;
parentId?: string;
name: string;
type: `${DatasetCollectionTypeEnum}`;
trainingType?: `${DatasetCollectionTrainingModeEnum}`;
chunkSize?: number;
fileId?: string;
rawLink?: string;
};
export type UpdateDatasetCollectionParams = {
id: string;
parentId?: string;

9
projects/app/src/global/core/dataset/api.d.ts vendored
View File

@ -1,5 +1,9 @@
import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api';
import { DatasetSearchModeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import {
DatasetSearchModeEnum,
DatasetTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constant';
import {
DatasetDataIndexItemType,
SearchDataResponseItemType
@ -8,8 +12,9 @@ import {
/* ================= dataset ===================== */
export type CreateDatasetParams = {
parentId?: string;
type: `${DatasetTypeEnum}`;
name: string;
tags: string;
intro: string;
avatar: string;
vectorModel?: string;
agentModel?: string;

47
projects/app/src/global/core/prompt/AIChat.ts
View File

@ -27,57 +27,56 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [
{
title: '标准模板',
desc: '',
value: `你的背景知识:
value: `你的知识库:
"""
{{quote}}
"""
对话要求：
1. 背景知识是最新的实时的信息，使用背景知识回答问题。
2. 优先使用背景知识的内容回答我的问题，答案应与背景知识严格一致。
3. 背景知识无法回答我的问题时，可以忽略背景知识，根据你的知识来自由回答。
4. 使用对话的风格，自然的回答问题。包含markdown内容，需按markdown格式返回。
回答要求：
1. 优先使用知识库内容回答问题。
2. 你可以回答我不知道。
3. 不要提及你是从知识库获取的知识。
4. 知识库包含 markdown 内容时，按 markdown 格式返回。
我的问题是:"{{question}}"`
},
{
title: '问答模板',
desc: '',
value: `你的背景知识:
value: `你的知识库:
"""
{{quote}}
"""
对话要求：
1. 背景知识是最新的实时的信息，使用背景知识回答问题，其中 instruction 是相关介绍，output 是预期回答或补充。
2. 优先使用背景知识的内容回答我的问题，答案应与背景知识严格一致。
3. 背景知识无法回答我的问题时，可以忽略背景知识，根据你的知识来自由回答。
4. 使用对话的风格，自然的回答问题。包含markdown内容，需按markdown格式返回。
回答要求：
1. 优先使用知识库内容回答问题，其中 instruction 是相关介绍，output 是预期回答或补充。
2. 你可以回答我不知道。
3. 不要提及你是从知识库获取的知识。
4. 知识库包含 markdown 内容时，按 markdown 格式返回。
我的问题是:"{{question}}"`
},
{
title: '标准严格模板',
desc: '',
value: `你的背景知识:
value: `你的知识库:
"""
{{quote}}
"""
对话要求：
1. 背景知识是最新的实时的信息，是你的唯一信息来源，使用背景知识回答问题。
2. 优先使用背景知识回答我的问题，答案与背景知识完全一致，无需做其他回答。
3. 背景知识与问题无关，或背景知识无法回答本次问题时，则拒绝回答本次问题：“我不太清除xxx”。
4. 使用对话的风格，自然的回答问题。包含markdown内容，需按markdown格式返回。
回答要求：
1. 仅使用知识库内容回答问题。
2. 与知识库无关的问题，你直接回答我不知道。
3. 不要提及你是从知识库获取的知识。
4. 知识库包含 markdown 内容时，按 markdown 格式返回。
我的问题是:"{{question}}"`
},
{
title: '严格问答模板',
desc: '',
value: `你的背景知识:
value: `你的知识库:
"""
{{quote}}
"""
对话要求：
1. 背景知识是最新的实时的信息，是你的唯一信息来源，使用背景知识回答问题。
2. 在背景知识的 JSON 中，question 是相关问题，answer 是已知答案。
3. 选择 answer 中的内容作为答案，要求答案与 answer 完全一致，无需做其他回答。
4. answer 中的答案无法满足问题，直接回复：“我不太清除xxx”。
回答要求：
1. 从知识库中选择一个合适的答案进行回答，其中 instruction 是相关问题，answer 是已知答案。
2. 与知识库无关的问题，你直接回答我不知道。
3. 不要提及你是从知识库获取的知识。
我的问题是:"{{question}}"`
}
];

2
projects/app/src/pages/account/components/BillDetail.tsx
View File

@ -35,7 +35,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void
<ModalBody>
<Flex alignItems={'center'} pb={4}>
<Box flex={'0 0 80px'}>用户:</Box>
<Box>{bill.memberName}</Box>
<Box>{t(bill.memberName)}</Box>
</Flex>
<Flex alignItems={'center'} pb={4}>
<Box flex={'0 0 80px'}>订单号:</Box>

4
projects/app/src/pages/account/components/Info.tsx
View File

@ -96,8 +96,8 @@ const UserInfo = () => {
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
onclickSave({

2
projects/app/src/pages/api/admin/initv46-2.ts
View File

@ -1,7 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '@fastgpt/service/common/pg';
import {
DatasetDataIndexTypeEnum,

7
projects/app/src/pages/api/admin/initv46-fix.ts
View File

@ -1,12 +1,9 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '@fastgpt/service/common/pg';
import {
DatasetDataIndexTypeEnum,
PgDatasetTableName
} from '@fastgpt/global/core/dataset/constant';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';

2
projects/app/src/pages/api/admin/initv46.ts
View File

@ -8,7 +8,7 @@ import {
} from '@fastgpt/service/support/user/team/controller';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { UserModelSchema } from '@fastgpt/global/support/user/type';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';

2
projects/app/src/pages/api/admin/initv462-2.ts
View File

@ -1,7 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';

20
projects/app/src/pages/api/admin/initv462.ts
View File

@ -1,7 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { jiebaSplit } from '@/service/core/dataset/utils';
@ -17,10 +17,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
console.log(
'total',
await MongoDatasetData.countDocuments({ fullTextToken: { $exists: false } })
await MongoDatasetData.countDocuments({
fullTextToken: { $exists: false },
updateTime: { $lt: new Date() }
})
);
await initFullTextToken(limit);
await initFullTextToken(limit, new Date());
jsonRes(res, {
message: 'success'
@ -34,9 +37,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
}
}
export async function initFullTextToken(limit = 50): Promise<any> {
export async function initFullTextToken(limit = 50, endDate: Date): Promise<any> {
try {
const dataList = await MongoDatasetData.find({ fullTextToken: { $exists: false } }, '_id q a')
const dataList = await MongoDatasetData.find(
{ fullTextToken: { $exists: false }, updateTime: { $lt: endDate } },
'_id q a'
)
.limit(limit)
.lean();
if (dataList.length === 0) return;
@ -56,9 +62,9 @@ export async function initFullTextToken(limit = 50): Promise<any> {
success += result.filter((item) => item.status === 'fulfilled').length;
console.log(`success: ${success}`);
return initFullTextToken(limit);
return initFullTextToken(limit, endDate);
} catch (error) {
await delay(1000);
return initFullTextToken(limit);
return initFullTextToken(limit, endDate);
}
}

62
projects/app/src/pages/api/admin/initv463-2.ts Normal file
View File

@ -0,0 +1,62 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { jiebaSplit } from '@/service/core/dataset/utils';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50 } = req.body as { limit: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
console.log('total', await MongoDatasetData.countDocuments({ inited: { $exists: false } }));
await initFullTextToken(limit);
jsonRes(res, {
message: 'success'
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function initFullTextToken(limit = 50): Promise<any> {
try {
const dataList = await MongoDatasetData.find({ inited: { $exists: false } }, '_id q a')
.limit(limit)
.lean();
if (dataList.length === 0) return;
const result = await Promise.allSettled(
dataList.map((item) => {
const text = item.q + (item.a || '');
const tokens = jiebaSplit({ text });
return MongoDatasetData.findByIdAndUpdate(item._id, {
$set: {
inited: true,
fullTextToken: tokens
}
});
})
);
success += result.filter((item) => item.status === 'fulfilled').length;
console.log(`success: ${success}`);
return initFullTextToken(limit);
} catch (error) {
await delay(1000);
return initFullTextToken(limit);
}
}

70
projects/app/src/pages/api/admin/initv463.ts
View File

@ -4,7 +4,8 @@ import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
@ -15,32 +16,85 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await connectToDatabase();
success = 0;
await MongoDatasetCollection.updateMany({}, [
await MongoDatasetCollection.updateMany({ createTime: { $exists: false } }, [
{
$set: {
createTime: '$updateTime'
}
}
]);
await MongoDatasetCollection.updateMany({ trainingType: { $exists: false } }, [
{
$set: {
createTime: '$updateTime',
trainingType: {
$cond: {
if: { $ifNull: ['$a', false] },
then: TrainingModeEnum.qa,
else: TrainingModeEnum.chunk
}
},
chunkSize: 0,
fileId: '$metadata.fileId',
}
}
}
]);
await MongoDatasetCollection.updateMany({ chunkSize: { $exists: false } }, [
{
$set: {
chunkSize: 0
}
}
]);
await MongoDatasetCollection.updateMany({ fileId: { $exists: false } }, [
{
$set: {
fileId: '$metadata.fileId'
}
}
]);
await MongoDatasetCollection.updateMany({ rawLink: { $exists: false } }, [
{
$set: {
rawLink: '$metadata.rawLink'
}
}
]);
await MongoDatasetData.updateMany(
{},
{ chunkIndex: { $exists: false } },
{
chunkIndex: 0
}
);
await MongoDatasetData.updateMany(
{ updateTime: { $exists: false } },
{
chunkIndex: 0,
updateTime: new Date()
}
);
await MongoDataset.updateMany(
{ status: { $exists: false } },
{
$set: {
status: DatasetStatusEnum.active
}
}
);
// dataset tags to intro
await MongoDataset.updateMany({ tags: { $exists: true } }, [
{
$set: {
intro: {
$reduce: {
input: '$tags',
initialValue: '',
in: { $concat: ['$$value', ' ', '$$this'] }
}
}
}
}
]);
jsonRes(res, {
message: 'success'
});

92
projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts Normal file
View File

@ -0,0 +1,92 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { delFileById, getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
import { addLog } from '@fastgpt/service/common/mongo/controller';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { delay } from '@fastgpt/global/common/system/utils';
/*
check dataset.files data. If there is no match in dataset.collections, delete it
*/
let deleteFileAmount = 0;
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const {
startDay = 10,
endDay = 3,
limit = 30
} = req.body as { startDay?: number; endDay?: number; limit?: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
// start: now - maxDay, end: now - 3 day
const start = new Date(Date.now() - startDay * 24 * 60 * 60 * 1000);
const end = new Date(Date.now() - endDay * 24 * 60 * 60 * 1000);
deleteFileAmount = 0;
checkFiles(start, end, limit);
jsonRes(res, {
message: 'success'
});
} catch (error) {
addLog.error(`check valid dataset files error`, error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function checkFiles(start: Date, end: Date, limit: number) {
const collection = getGFSCollection('dataset');
const where = {
uploadDate: { $gte: start, $lte: end }
};
// 1. get all _id
const ids = await collection
.find(where, {
projection: {
_id: 1
}
})
.toArray();
console.log('total files', ids.length);
for (let i = 0; i < limit; i++) {
check(i);
}
async function check(index: number): Promise<any> {
const id = ids[index];
if (!id) {
console.log(`检测完成，共删除 ${deleteFileAmount} 个无效文件`);
return;
}
try {
const { _id } = id;
// 2. find fileId in dataset.collections
const hasCollection = await MongoDatasetCollection.countDocuments({ fileId: _id });
// 3. if not found, delete file
if (hasCollection === 0) {
await delFileById({ bucketName: 'dataset', fileId: String(_id) });
console.log('delete file', _id);
deleteFileAmount++;
}
index % 100 === 0 && console.log(index);
return check(index + limit);
} catch (error) {
console.log(error);
await delay(2000);
return check(index);
}
}
}

33
projects/app/src/pages/api/common/file/read.ts
View File

@ -2,8 +2,8 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authFileToken } from '@fastgpt/service/support/permission/controller';
import jschardet from 'jschardet';
import { getDownloadBuf, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
import { detect } from 'jschardet';
import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@ -11,24 +11,43 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { token } = req.query as { token: string };
const { fileId, teamId, tmbId, bucketName } = await authFileToken(token);
const { fileId, bucketName } = await authFileToken(token);
if (!fileId) {
throw new Error('fileId is empty');
}
const [file, buffer] = await Promise.all([
const [file, encodeStream] = await Promise.all([
getFileById({ bucketName, fileId }),
getDownloadBuf({ bucketName, fileId })
getDownloadStream({ bucketName, fileId })
]);
const encoding = jschardet.detect(buffer)?.encoding;
// get encoding
let buffers: Buffer = Buffer.from([]);
for await (const chunk of encodeStream) {
buffers = Buffer.concat([buffers, chunk]);
if (buffers.length > 10) {
encodeStream.abort();
break;
}
}
const encoding = detect(buffers)?.encoding || 'utf-8';
res.setHeader('Content-Type', `${file.contentType}; charset=${encoding}`);
res.setHeader('Cache-Control', 'public, max-age=3600');
res.setHeader('Content-Disposition', `inline; filename="${encodeURIComponent(file.filename)}"`);
res.end(buffer);
const fileStream = await getDownloadStream({ bucketName, fileId });
fileStream.pipe(res);
fileStream.on('error', () => {
res.status(500).end();
});
fileStream.on('end', () => {
res.end();
});
} catch (error) {
jsonRes(res, {
code: 500,

6
projects/app/src/pages/api/core/chat/chatTest.ts
View File

@ -52,12 +52,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
/* start process */
const { responseData } = await dispatchModules({
res,
appId,
modules,
variables,
teamId,
tmbId,
user,
appId,
modules,
variables,
params: {
history,
userChatInput: prompt

18
projects/app/src/pages/api/core/dataset/allDataset.ts
View File

@ -2,10 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import { getVectorModel } from '@/service/core/ai/model';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
/* get all dataset by teamId or tmbId */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@ -16,18 +17,23 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const datasets = await MongoDataset.find({
...mongoRPermission({ teamId, tmbId, role }),
type: 'dataset'
type: { $ne: DatasetTypeEnum.folder }
}).lean();
const data = datasets.map((item) => ({
...item,
_id: item._id,
parentId: item.parentId,
avatar: item.avatar,
name: item.name,
intro: item.intro,
type: item.type,
permission: item.permission,
vectorModel: getVectorModel(item.vectorModel),
agentModel: getQAModel(item.agentModel),
canWrite: String(item.tmbId) === tmbId,
isOwner: teamOwner || String(item.tmbId) === tmbId
}));
jsonRes<DatasetItemType[]>(res, {
jsonRes<DatasetListItemType[]>(res, {
data
});
} catch (err) {

74
projects/app/src/pages/api/core/dataset/collection/create.ts
View File

@ -4,15 +4,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { CreateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum,
DatasetCollectionTrainingModeEnum
} from '@fastgpt/global/core/dataset/constant';
import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@ -42,68 +37,3 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
}
}
export async function createOneCollection({
name,
parentId,
datasetId,
type,
trainingType = DatasetCollectionTrainingModeEnum.manual,
chunkSize = 0,
fileId,
rawLink,
teamId,
tmbId
}: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
const { _id } = await MongoDatasetCollection.create({
name,
teamId,
tmbId,
datasetId,
parentId: parentId || null,
type,
trainingType,
chunkSize,
fileId,
rawLink
});
// create default collection
if (type === DatasetCollectionTypeEnum.folder) {
await createDefaultCollection({
datasetId,
parentId: _id,
teamId,
tmbId
});
}
return _id;
}
// create default collection
export function createDefaultCollection({
name = '手动录入',
datasetId,
parentId,
teamId,
tmbId
}: {
name?: '手动录入' | '手动标注';
datasetId: string;
parentId?: string;
teamId: string;
tmbId: string;
}) {
return MongoDatasetCollection.create({
name,
teamId,
tmbId,
datasetId,
parentId,
type: DatasetCollectionTypeEnum.virtual,
trainingType: DatasetCollectionTrainingModeEnum.manual,
chunkSize: 0,
updateTime: new Date('2099')
});
}

31
projects/app/src/pages/api/core/dataset/collection/delById.ts → projects/app/src/pages/api/core/dataset/collection/delete.ts
View File

@ -1,13 +1,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils';
import { delDataByCollectionId } from '@/service/core/dataset/data/controller';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { delFileById } from '@fastgpt/service/common/file/gridfs/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@ -19,7 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('CollectionIdId is required');
}
const { teamId } = await authDatasetCollection({
await authDatasetCollection({
req,
authToken: true,
collectionId,
@ -30,26 +27,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const collections = await findCollectionAndChild(collectionId, '_id metadata');
const delIdList = collections.map((item) => item._id);
// delete training data
await MongoDatasetTraining.deleteMany({
collectionId: { $in: delIdList },
teamId
// delete
await delCollectionRelevantData({
collectionIds: delIdList,
fileIds: collections.map((item) => String(item.metadata?.fileId)).filter(Boolean)
});
// delete pg data
await delDataByCollectionId({ collectionIds: delIdList });
// delete file
await Promise.all(
collections.map((collection) => {
if (!collection?.fileId) return;
return delFileById({
bucketName: BucketNameEnum.dataset,
fileId: collection.fileId
});
})
);
// delete collection
await MongoDatasetCollection.deleteMany({
_id: { $in: delIdList }

11
projects/app/src/pages/api/core/dataset/collection/list.ts
View File

@ -6,7 +6,10 @@ import { Types } from '@fastgpt/service/common/mongo';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
import { PagingData } from '@/types';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import {
DatasetColCollectionName,
MongoDatasetCollection
} from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { startQueue } from '@/service/utils/tools';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
@ -45,7 +48,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// not count data amount
if (simple) {
const collections = await MongoDatasetCollection.find(match, '_id name type parentId')
const collections = await MongoDatasetCollection.find(match, '_id parentId type name')
.sort({
updateTime: -1
})
@ -72,6 +75,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
{
$match: match
},
// count training data
{
$lookup: {
from: DatasetTrainingCollectionName,
@ -89,6 +93,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
as: 'trainings'
}
},
// count collection total data
{
$lookup: {
from: DatasetDataCollectionName,
@ -106,7 +111,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
as: 'datas'
}
},
// 统计子集合的数量和子训练的数量
{
$project: {
_id: 1,
@ -114,6 +118,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
tmbId: 1,
name: 1,
type: 1,
status: 1,
updateTime: 1,
dataAmount: { $size: '$datas' },
trainingAmount: { $size: '$trainings' },

23
projects/app/src/pages/api/core/dataset/create.ts
View File

@ -3,20 +3,20 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import { createDefaultCollection } from './collection/create';
import { createDefaultCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
parentId,
name,
tags,
type,
avatar,
vectorModel = global.vectorModels[0].model,
agentModel,
parentId,
type
agentModel
} = req.body as CreateDatasetParams;
// 凭证校验
@ -26,7 +26,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
name,
teamId,
tmbId,
tags,
vectorModel,
agentModel,
avatar,
@ -34,11 +33,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
type
});
await createDefaultCollection({
datasetId: _id,
teamId,
tmbId
});
if (type === DatasetTypeEnum.dataset) {
await createDefaultCollection({
datasetId: _id,
teamId,
tmbId
});
}
jsonRes(res, { data: _id });
} catch (err) {

4
projects/app/src/pages/api/core/dataset/data/delete.ts
View File

@ -3,7 +3,7 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import { deleteDataByDataId } from '@/service/core/dataset/data/controller';
import { delDatasetDataByDataId } from '@fastgpt/service/core/dataset/data/controller';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@ -19,7 +19,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// 凭证校验
await authDatasetData({ req, authToken: true, dataId, per: 'w' });
await deleteDataByDataId(dataId);
await delDatasetDataByDataId(dataId);
jsonRes(res, {
data: 'success'

22
projects/app/src/pages/api/core/dataset/delete.ts
View File

@ -1,13 +1,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { delDatasetFiles } from '@fastgpt/service/core/dataset/file/controller';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { delDataByDatasetId } from '@/service/core/dataset/data/controller';
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@ -25,21 +22,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const deletedIds = await findDatasetIdTreeByTopDatasetId(id);
// delete training data(There could be a training mission)
await MongoDatasetTraining.deleteMany({
datasetId: { $in: deletedIds }
});
// delete all dataset.data and pg data
await delDataByDatasetId({ datasetIds: deletedIds });
// delete related files
await delDatasetFiles({ datasetId: id });
// delete collections
await MongoDatasetCollection.deleteMany({
datasetId: { $in: deletedIds }
});
await delDatasetRelevantData({ datasetIds: deletedIds });
// delete dataset data
await MongoDataset.deleteMany({

21
projects/app/src/pages/api/core/dataset/list.ts
View File

@ -1,12 +1,12 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
import { getVectorModel } from '@/service/core/ai/model';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@ -27,16 +27,21 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
.lean();
const data = await Promise.all(
datasets.map(async (item) => ({
...item,
vectorModel: getVectorModel(item.vectorModel),
agentModel: getQAModel(item.agentModel),
datasets.map<DatasetListItemType>((item) => ({
_id: item._id,
parentId: item.parentId,
avatar: item.avatar,
name: item.name,
intro: item.intro,
type: item.type,
permission: item.permission,
canWrite,
isOwner: teamOwner || String(item.tmbId) === tmbId
isOwner: teamOwner || String(item.tmbId) === tmbId,
vectorModel: getVectorModel(item.vectorModel)
}))
);
jsonRes<DatasetItemType[]>(res, {
jsonRes<DatasetListItemType[]>(res, {
data
});
} catch (err) {

8
projects/app/src/pages/api/core/dataset/searchTest.ts
View File

@ -44,7 +44,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
// push bill
pushGenerateVectorBill({
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
@ -54,11 +54,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
if (apikey) {
updateApiKeyUsage({
apikey,
usage: countModelPrice({
model: dataset.vectorModel,
tokens: tokenLen,
type: ModelTypeEnum.vector
})
usage: total
});
}

10
projects/app/src/pages/api/core/dataset/update.ts
View File

@ -2,14 +2,14 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { DatasetUpdateParams } from '@/global/core/api/datasetReq.d';
import type { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, parentId, name, avatar, tags, permission, agentModel } =
req.body as DatasetUpdateParams;
const { id, parentId, name, avatar, tags, permission, agentModel, websiteConfig, status } =
req.body as DatasetUpdateBody;
if (!id) {
throw new Error('缺少参数');
@ -28,7 +28,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
...(avatar && { avatar }),
...(tags && { tags }),
...(permission && { permission }),
...(agentModel && { agentModel: agentModel.model })
...(agentModel && { agentModel: agentModel.model }),
...(websiteConfig && { websiteConfig }),
...(status && { status })
}
);

73
projects/app/src/pages/api/plugins/urlFetch.ts
View File

@ -1,73 +0,0 @@
// pages/api/fetchContent.ts
import { NextApiRequest, NextApiResponse } from 'next';
import axios from 'axios';
import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
import { simpleText } from '@fastgpt/global/common/string/tools';
import { connectToDatabase } from '@/service/mongo';
export type UrlFetchResponse = FetchResultItem[];
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
try {
await connectToDatabase();
let { urlList = [] } = req.body as { urlList: string[] };
if (!urlList || urlList.length === 0) {
throw new Error('urlList is empty');
}
await authCert({ req, authToken: true });
urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
const response = (
await Promise.allSettled(
urlList.map(async (url) => {
try {
const fetchRes = await axios.get(url, {
timeout: 30000
});
const dom = new JSDOM(fetchRes.data, {
url,
contentType: 'text/html'
});
const reader = new Readability(dom.window.document);
const article = reader.parse();
const content = article?.textContent || '';
return {
url,
content: simpleText(`${article?.title}\n${content}`)
};
} catch (error) {
return {
url,
content: ''
};
}
})
)
)
.filter((item) => item.status === 'fulfilled')
.map((item: any) => item.value)
.filter((item) => item.content);
jsonRes<UrlFetchResponse>(res, {
data: response
});
} catch (error: any) {
jsonRes(res, {
code: 500,
error: error
});
}
};
export default fetchContent;

4
projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts
View File

@ -24,13 +24,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
source: BillSourceEnum.training,
list: [
{
moduleName: '索引生成',
moduleName: 'wallet.moduleName.index',
model: vectorModelData.name,
amount: 0,
tokenLen: 0
},
{
moduleName: 'QA 拆分',
moduleName: 'wallet.moduleName.qa',
model: agentModelData.name,
amount: 0,
tokenLen: 0

34
projects/app/src/pages/api/tools/urlFetch.ts Normal file
View File

@ -0,0 +1,34 @@
// pages/api/fetchContent.ts
import { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { connectToDatabase } from '@/service/mongo';
import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
import { urlsFetch } from '@fastgpt/global/common/file/tools';
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
try {
await connectToDatabase();
let { urlList = [], selector } = req.body as UrlFetchParams;
if (!urlList || urlList.length === 0) {
throw new Error('urlList is empty');
}
await authCert({ req, authToken: true });
jsonRes<UrlFetchResponse>(res, {
data: await urlsFetch({
urlList,
selector
})
});
} catch (error: any) {
jsonRes(res, {
code: 500,
error: error
});
}
};
export default fetchContent;

8
projects/app/src/pages/api/v1/chat/completions.ts
View File

@ -14,10 +14,10 @@ import { getChatHistory } from './getHistory';
import { saveChat } from '@/service/utils/chat/saveChat';
import { responseWrite } from '@fastgpt/service/common/response';
import { pushChatBill } from '@/service/support/wallet/bill/push';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { authOutLinkChat } from '@/service/support/permission/auth/outLink';
import { pushResult2Remote, updateOutLinkUsage } from '@fastgpt/service/support/outLink/tools';
import requestIp from 'request-ip';
import { getBillSourceByAuthType } from '@fastgpt/global/support/wallet/bill/tools';
import { selectShareResponse } from '@/utils/service/core/chat';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
@ -276,11 +276,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
appId: app._id,
teamId: user.team.teamId,
tmbId: user.team.tmbId,
source: (() => {
if (authType === 'apikey') return BillSourceEnum.api;
if (shareId) return BillSourceEnum.shareLink;
return BillSourceEnum.fastgpt;
})(),
source: getBillSourceByAuthType({ shareId, authType }),
response: responseData
});

35
projects/app/src/pages/api/v1/embeddings.ts
View File

@ -6,6 +6,8 @@ import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
import { connectToDatabase } from '@/service/mongo';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { getVectorsByText, GetVectorProps } from '@/service/core/ai/vector';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
import { getBillSourceByAuthType } from '@fastgpt/global/support/wallet/bill/tools';
type Props = GetVectorProps & {
billId?: string;
@ -15,24 +17,21 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
try {
let { input, model, billId } = req.body as Props;
await connectToDatabase();
const { teamId, tmbId } = await authCert({ req, authToken: true, authApiKey: true });
if (!Array.isArray(input) || typeof input !== 'string') {
if (!Array.isArray(input) && typeof input !== 'string') {
throw new Error('input is nor array or string');
}
const { teamId, tmbId, apikey, authType } = await authCert({
req,
authToken: true,
authApiKey: true
});
await authTeamBalance(teamId);
const { tokenLen, vectors } = await getVectorsByText({ input, model });
pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
model,
billId
});
jsonRes(res, {
data: {
object: 'list',
@ -48,6 +47,22 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
}
}
});
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokenLen,
model,
billId,
source: getBillSourceByAuthType({ authType })
});
if (apikey) {
updateApiKeyUsage({
apikey,
usage: total
});
}
} catch (err) {
console.log(err);
jsonRes(res, {

12
projects/app/src/pages/api/v1/rerank.ts
View File

@ -7,12 +7,13 @@ import { connectToDatabase } from '@/service/mongo';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { PostReRankProps, PostReRankResponse } from '@fastgpt/global/core/ai/api';
import { reRankRecall } from '@/service/core/ai/rerank';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let { query, inputs } = req.body as PostReRankProps;
try {
await connectToDatabase();
const { teamId, tmbId } = await authCert({
const { teamId, tmbId, apikey } = await authCert({
req,
authApiKey: true
});
@ -23,12 +24,19 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
const result = await reRankRecall({ query, inputs });
pushReRankBill({
const { total } = pushReRankBill({
teamId,
tmbId,
source: 'api'
});
if (apikey) {
updateApiKeyUsage({
apikey,
usage: total
});
}
jsonRes<PostReRankResponse>(res, {
data: result
});

4
projects/app/src/pages/app/detail/components/InfoModal.tsx
View File

@ -103,8 +103,8 @@ const InfoModal = ({
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
setRefresh((state) => !state);

4
projects/app/src/pages/app/list/component/CreateModal.tsx
View File

@ -60,8 +60,8 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
setRefresh((state) => !state);

366
projects/app/src/pages/dataset/detail/components/CollectionCard.tsx
View File

@ -11,14 +11,18 @@ import {
Tbody,
Image,
MenuButton,
useDisclosure
useDisclosure,
Button,
Link,
useTheme
} from '@chakra-ui/react';
import {
getDatasetCollections,
delDatasetCollectionById,
putDatasetCollectionById,
postDatasetCollection,
getDatasetCollectionPathById
getDatasetCollectionPathById,
postWebsiteSync
} from '@/web/core/dataset/api';
import { useQuery } from '@tanstack/react-query';
import { debounce } from 'lodash';
@ -39,7 +43,10 @@ import EmptyTip from '@/components/EmptyTip';
import {
FolderAvatarSrc,
DatasetCollectionTypeEnum,
TrainingModeEnum
DatasetCollectionTrainingModeEnum,
DatasetTypeEnum,
DatasetTypeMap,
DatasetStatusEnum
} from '@fastgpt/global/core/dataset/constant';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
@ -52,13 +59,18 @@ import { useToast } from '@/web/common/hooks/useToast';
import MyTooltip from '@/components/MyTooltip';
import { useUserStore } from '@/web/support/user/useUserStore';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
const FileImportModal = dynamic(() => import('./Import/ImportModal'), {});
const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {});
const CollectionCard = () => {
const BoxRef = useRef<HTMLDivElement>(null);
const lastSearch = useRef('');
const router = useRouter();
const theme = useTheme();
const { toast } = useToast();
const { parentId = '', datasetId } = router.query as { parentId: string; datasetId: string };
const { t } = useTranslation();
@ -66,7 +78,7 @@ const CollectionCard = () => {
const { isPc } = useSystemStore();
const { userInfo } = useUserStore();
const [searchText, setSearchText] = useState('');
const { setLoading } = useSystemStore();
const { datasetDetail, updateDataset, loadDatasetDetail } = useDatasetStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: t('dataset.Confirm to delete the file')
@ -76,11 +88,18 @@ const CollectionCard = () => {
onOpen: onOpenFileImportModal,
onClose: onCloseFileImportModal
} = useDisclosure();
const {
isOpen: isOpenWebsiteModal,
onOpen: onOpenWebsiteModal,
onClose: onCloseWebsiteModal
} = useDisclosure();
const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } =
useEditTitle({
title: t('dataset.Create Virtual File'),
tip: t('dataset.Virtual File Tip')
tip: t('dataset.Virtual File Tip'),
canEmpty: false
});
const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
title: t('Rename')
});
@ -128,48 +147,60 @@ const CollectionCard = () => {
() =>
collections.map((collection) => {
const icon = getCollectionIcon(collection.type, collection.name);
const status = (() => {
if (collection.trainingAmount > 0) {
return {
statusText: t('dataset.collections.Collection Embedding', {
total: collection.trainingAmount
}),
color: 'myGray.500'
};
}
return {
statusText: t('core.dataset.collection.status.active'),
color: 'green.500'
};
})();
return {
...collection,
icon,
...(collection.trainingAmount > 0
? {
statusText: t('dataset.collections.Collection Embedding', {
total: collection.trainingAmount
}),
color: 'myGray.500'
}
: {
statusText: t('dataset.collections.Ready'),
color: 'green.500'
})
...status
};
}),
[collections, t]
);
const hasTrainingData = useMemo(
() => !!formatCollections.find((item) => item.trainingAmount > 0),
[formatCollections]
);
const { mutate: onCreateVirtualFile } = useRequest({
mutationFn: ({ name }: { name: string }) => {
setLoading(true);
return postDatasetCollection({
const { mutate: onCreateCollection, isLoading: isCreating } = useRequest({
mutationFn: async ({
name,
type,
callback,
...props
}: {
name: string;
type: `${DatasetCollectionTypeEnum}`;
callback?: (id: string) => void;
trainingType?: `${DatasetCollectionTrainingModeEnum}`;
rawLink?: string;
chunkSize?: number;
}) => {
const id = await postDatasetCollection({
parentId,
datasetId,
name,
type: DatasetCollectionTypeEnum.virtual
type,
...props
});
callback?.(id);
return id;
},
onSuccess() {
getData(pageNum);
},
onSettled() {
setLoading(false);
},
successToast: t('dataset.collections.Create Virtual File Success'),
errorToast: t('common.Create Virtual File Failed')
successToast: t('common.Create Success'),
errorToast: t('common.Create Failed')
});
const { mutate: onUpdateCollectionName } = useRequest({
mutationFn: ({ collectionId, name }: { collectionId: string; name: string }) => {
@ -185,9 +216,8 @@ const CollectionCard = () => {
successToast: t('common.Rename Success'),
errorToast: t('common.Rename Failed')
});
const { mutate: onDelCollection } = useRequest({
const { mutate: onDelCollection, isLoading: isDeleting } = useRequest({
mutationFn: (collectionId: string) => {
setLoading(true);
return delDatasetCollectionById({
collectionId
});
@ -195,26 +225,54 @@ const CollectionCard = () => {
onSuccess() {
getData(pageNum);
},
onSettled() {
setLoading(false);
},
successToast: t('common.Delete Success'),
errorToast: t('common.Delete Failed')
});
const { mutate: onUpdateDatasetWebsiteConfig, isLoading: isUpdating } = useRequest({
mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => {
onCloseWebsiteModal();
const [_, billId] = await Promise.all([
updateDataset({
id: datasetDetail._id,
websiteConfig,
status: DatasetStatusEnum.syncing
}),
postCreateTrainingBill({
name: 'core.dataset.training.Website Sync',
vectorModel: datasetDetail.vectorModel.model,
agentModel: datasetDetail.agentModel.model
})
]);
return billId;
},
onSuccess(billId: string) {
try {
postWebsiteSync({ datasetId: datasetDetail._id, billId });
} catch (error) {}
},
errorToast: t('common.Update Failed')
});
const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
getDatasetCollectionPathById(parentId)
);
const hasTrainingData = useMemo(
() => !!formatCollections.find((item) => item.trainingAmount > 0),
[formatCollections]
);
useQuery(
['refreshCollection'],
() => {
getData(1);
if (datasetDetail.status === DatasetStatusEnum.syncing) {
loadDatasetDetail(datasetId, true);
}
return null;
},
{
refetchInterval: 6000,
enabled: hasTrainingData
enabled: hasTrainingData || datasetDetail.status === DatasetStatusEnum.syncing
}
);
@ -224,17 +282,33 @@ const CollectionCard = () => {
return (
<Flex flexDirection={'column'} ref={BoxRef} py={[1, 3]} h={'100%'}>
<Flex px={[2, 5]} alignItems={['flex-start', 'center']}>
<Flex px={[2, 5]} alignItems={['flex-start', 'center']} h={'35px'}>
<Box flex={1}>
<ParentPath
paths={paths.map((path, i) => ({
parentId: path.parentId,
parentName: i === paths.length - 1 ? `${path.parentName}(${total})` : path.parentName
parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName
}))}
FirstPathDom={
<Box fontWeight={'bold'} fontSize={['sm', 'lg']}>
{t('common.File')}({total})
</Box>
<>
<Box fontWeight={'bold'} fontSize={['sm', 'lg']}>
{t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total})
</Box>
{datasetDetail?.websiteConfig?.url && (
<Flex fontSize={'sm'}>
{t('core.dataset.website.Base Url')}:
<Link
href={datasetDetail.websiteConfig.url}
target="_blank"
mr={2}
textDecoration={'underline'}
color={'myBlue.700'}
>
{datasetDetail.websiteConfig.url}
</Link>
</Flex>
)}
</>
}
onClick={(e) => {
router.replace({
@ -279,68 +353,109 @@ const CollectionCard = () => {
/>
</Flex>
)}
{userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
<MyMenu
offset={[-40, 10]}
width={120}
Button={
<MenuButton
_hover={{
color: 'myBlue.600'
}}
fontSize={['sm', 'md']}
>
<Flex
alignItems={'center'}
px={5}
py={2}
borderRadius={'md'}
cursor={'pointer'}
bg={'myBlue.600'}
overflow={'hidden'}
color={'white'}
h={['28px', '35px']}
>
<MyIcon name={'importLight'} mr={2} w={'14px'} />
<Box>{t('dataset.collections.Create And Import')}</Box>
</Flex>
</MenuButton>
}
menuList={[
{
child: (
<Flex>
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={2} />
{t('Folder')}
</Flex>
),
onClick: () => setEditFolderData({})
},
{
child: (
<Flex>
<Image src={'/imgs/files/collection.svg'} alt={''} w={'20px'} mr={2} />
{t('dataset.Create Virtual File')}
</Flex>
),
onClick: () => {
onOpenCreateVirtualFileModal({
defaultVal: '',
onSuccess: (name) => onCreateVirtualFile({ name })
});
{datasetDetail?.type === DatasetTypeEnum.dataset && (
<>
{userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
<MyMenu
offset={[-40, 10]}
width={120}
Button={
<MenuButton
_hover={{
color: 'myBlue.600'
}}
fontSize={['sm', 'md']}
>
<Flex
alignItems={'center'}
px={5}
py={2}
borderRadius={'md'}
cursor={'pointer'}
bg={'myBlue.600'}
overflow={'hidden'}
color={'white'}
h={['28px', '35px']}
>
<MyIcon name={'importLight'} mr={2} w={'14px'} />
<Box>{t('dataset.collections.Create And Import')}</Box>
</Flex>
</MenuButton>
}
},
{
child: (
<Flex>
<Image src={'/imgs/files/file.svg'} alt={''} w={'20px'} mr={2} />
{t('dataset.File Input')}
menuList={[
{
child: (
<Flex>
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={2} />
{t('Folder')}
</Flex>
),
onClick: () => setEditFolderData({})
},
{
child: (
<Flex>
<Image src={'/imgs/files/collection.svg'} alt={''} w={'20px'} mr={2} />
{t('dataset.Create Virtual File')}
</Flex>
),
onClick: () => {
onOpenCreateVirtualFileModal({
defaultVal: '',
onSuccess: (name) => {
onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual });
}
});
}
},
{
child: (
<Flex>
<Image src={'/imgs/files/file.svg'} alt={''} w={'20px'} mr={2} />
{t('dataset.File Input')}
</Flex>
),
onClick: onOpenFileImportModal
}
]}
/>
)}
</>
)}
{datasetDetail?.type === DatasetTypeEnum.websiteDataset && (
<>
{datasetDetail?.websiteConfig?.url ? (
<Flex alignItems={'center'}>
{datasetDetail.status === DatasetStatusEnum.active && (
<Button onClick={onOpenWebsiteModal}>{t('common.Config')}</Button>
)}
{datasetDetail.status === DatasetStatusEnum.syncing && (
<Flex
ml={3}
alignItems={'center'}
px={3}
py={1}
borderRadius="md"
border={theme.borders.base}
>
<Box
animation={'zoomStopIcon 0.5s infinite alternate'}
bg={'myGray.700'}
w="8px"
h="8px"
borderRadius={'50%'}
mt={'1px'}
></Box>
<Box ml={2} color={'myGray.600'}>
{t('core.dataset.status.syncing')}
</Box>
</Flex>
),
onClick: onOpenFileImportModal
}
]}
/>
)}
</Flex>
) : (
<Button onClick={onOpenWebsiteModal}>{t('core.dataset.Set Website Config')}</Button>
)}
</>
)}
</Flex>
@ -428,11 +543,7 @@ const CollectionCard = () => {
</MyTooltip>
</Flex>
</Td>
<Td fontSize={'md'}>
{collection.type === DatasetCollectionTypeEnum.folder
? '-'
: collection.dataAmount}
</Td>
<Td fontSize={'md'}>{collection.dataAmount || '-'}</Td>
<Td>{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td>
<Flex
@ -443,10 +554,10 @@ const CollectionCard = () => {
h: '10px',
mr: 2,
borderRadius: 'lg',
bg: collection?.color
bg: collection.color
}}
>
{collection?.statusText}
{t(collection.statusText)}
</Flex>
</Td>
<Td onClick={(e) => e.stopPropagation()}>
@ -536,14 +647,31 @@ const CollectionCard = () => {
))}
</Tbody>
</Table>
<Loading loading={isLoading && collections.length === 0} fixed={false} />
{total > pageSize && (
<Flex mt={2} justifyContent={'center'}>
<Pagination />
</Flex>
)}
{total === 0 && <EmptyTip text="数据集空空如也" />}
{total === 0 && (
<EmptyTip
text={
datasetDetail.type === DatasetTypeEnum.dataset ? (
t('core.dataset.collection.Empty Tip')
) : (
<Flex>
{t('core.dataset.collection.Website Empty Tip')}
<Box textDecoration={'underline'} cursor={'pointer'} onClick={onOpenWebsiteModal}>
{t('core.dataset.collection.Click top config website')}
</Box>
</Flex>
)
}
/>
)}
</TableContainer>
<Loading
loading={isCreating || isDeleting || isUpdating || (isLoading && collections.length === 0)}
/>
<ConfirmModal />
<EditTitleModal />
@ -559,7 +687,6 @@ const CollectionCard = () => {
onClose={onCloseFileImportModal}
/>
)}
{!!editFolderData && (
<EditFolderModal
onClose={() => setEditFolderData(undefined)}
@ -570,15 +697,13 @@ const CollectionCard = () => {
id: editFolderData.id,
name
});
getData(pageNum);
} else {
await postDatasetCollection({
parentId,
datasetId,
onCreateCollection({
name,
type: DatasetCollectionTypeEnum.folder
});
}
getData(pageNum);
} catch (error) {
return Promise.reject(error);
}
@ -587,7 +712,6 @@ const CollectionCard = () => {
name={editFolderData.name}
/>
)}
{!!moveCollectionData && (
<SelectCollections
datasetId={datasetId}
@ -608,6 +732,16 @@ const CollectionCard = () => {
}}
/>
)}
{isOpenWebsiteModal && (
<WebSiteConfigModal
onClose={onCloseWebsiteModal}
onSuccess={onUpdateDatasetWebsiteConfig}
defaultValue={{
url: datasetDetail?.websiteConfig?.url,
selector: datasetDetail?.websiteConfig?.selector
}}
/>
)}
</Flex>
);
};

20
projects/app/src/pages/dataset/detail/components/DataCard.tsx
View File

@ -55,7 +55,10 @@ const DataCard = () => {
const router = useRouter();
const { userInfo } = useUserStore();
const { isPc } = useSystemStore();
const { collectionId = '' } = router.query as { collectionId: string };
const { collectionId = '', datasetId } = router.query as {
collectionId: string;
datasetId: string;
};
const { Loading, setIsLoading } = useLoading({ defaultLoading: true });
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
@ -99,8 +102,18 @@ const DataCard = () => {
);
// get file info
const { data: collection } = useQuery(['getDatasetCollectionById', collectionId], () =>
getDatasetCollectionById(collectionId)
const { data: collection } = useQuery(
['getDatasetCollectionById', collectionId],
() => getDatasetCollectionById(collectionId),
{
onError: () => {
router.replace({
query: {
datasetId
}
});
}
}
);
const canWrite = useMemo(
@ -290,6 +303,7 @@ const DataCard = () => {
</Flex>
<Box
maxH={'135px'}
minH={'90px'}
overflow={'hidden'}
wordBreak={'break-all'}
pt={1}

4
projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
View File

@ -18,13 +18,13 @@ import { useTranslation } from 'next-i18next';
import { customAlphabet } from 'nanoid';
import dynamic from 'next/dynamic';
import MyTooltip from '@/components/MyTooltip';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';
import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
const CreateFileModal = dynamic(() => import('./CreateFileModal'));
@ -215,7 +215,7 @@ const FileSelect = ({
);
// link fetch
const onUrlFetch = useCallback(
(e: FetchResultItem[]) => {
(e: UrlFetchResponse) => {
const result: FileItemType[] = e.map<FileItemType>(({ url, content }) => {
const splitRes = splitText2Chunks({
text: content,

3
projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
View File

@ -188,6 +188,8 @@ const Provider = ({
const onReSplitChunks = useCallback(async () => {
try {
setPreviewFile(undefined);
setFiles((state) =>
state.map((file) => {
const splitRes = splitText2Chunks({
@ -490,6 +492,7 @@ export const SelectorContainer = ({
display={['block', 'none']}
onClick={(e) => {
e.stopPropagation();
setPreviewFile(undefined);
setFiles((state) => state.filter((file) => file.id !== item.id));
}}
/>

65
projects/app/src/pages/dataset/detail/components/Import/UrlFetchModal.tsx
View File

@ -1,31 +1,39 @@
import React, { useRef } from 'react';
import React from 'react';
import { useTranslation } from 'next-i18next';
import MyModal from '@/components/MyModal';
import { Box, Button, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
import { Box, Button, Input, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
import { useRequest } from '@/web/common/hooks/useRequest';
import { postFetchUrls } from '@/web/common/plugin/api';
import { postFetchUrls } from '@/web/common/tools/api';
import { useForm } from 'react-hook-form';
import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
const UrlFetchModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: (e: FetchResultItem[]) => void;
onSuccess: (e: UrlFetchResponse) => void;
}) => {
const { t } = useTranslation();
const Dom = useRef<HTMLTextAreaElement>(null);
const { register, handleSubmit } = useForm({
defaultValues: {
urls: '',
selector: ''
}
});
const { mutate, isLoading } = useRequest({
mutationFn: async () => {
const val = Dom.current?.value || '';
const urls = val.split('\n').filter((e) => e);
const res = await postFetchUrls(urls);
mutationFn: async ({ urls, selector }: { urls: string; selector: string }) => {
const urlList = urls.split('\n').filter((e) => e);
const res = await postFetchUrls({
urlList,
selector
});
onSuccess(res);
onClose();
},
errorToast: '获取链接失败'
errorToast: t('core.dataset.import.Fetch Error')
});
return (
@ -34,8 +42,8 @@ const UrlFetchModal = ({
title={
<Box>
<Box>{t('file.Fetch Url')}</Box>
<Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'} mt={1}>
目前仅支持读取静态链接，请注意检查结果
<Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'}>
{t('core.dataset.import.Fetch url tip')}
</Box>
</Box>
}
@ -45,20 +53,31 @@ const UrlFetchModal = ({
w={'600px'}
>
<ModalBody>
<Textarea
ref={Dom}
rows={12}
whiteSpace={'nowrap'}
resize={'both'}
placeholder={'最多10个链接，每行一个。'}
/>
<Box>
<Box fontWeight={'bold'}>{t('core.dataset.import.Fetch Url')}</Box>
<Textarea
{...register('urls', {
required: true
})}
rows={11}
whiteSpace={'nowrap'}
resize={'both'}
placeholder={t('core.dataset.import.Fetch url placeholder')}
/>
</Box>
<Box mt={4}>
<Box fontWeight={'bold'}>
{t('core.dataset.website.Selector')}({t('common.choosable')})
</Box>{' '}
<Input {...register('selector')} placeholder="body .content #document" />
</Box>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
取消
{t('common.Close')}
</Button>
<Button isLoading={isLoading} onClick={mutate}>
确认
<Button isLoading={isLoading} onClick={handleSubmit((data) => mutate(data))}>
{t('common.Confirm')}
</Button>
</ModalFooter>
</MyModal>

101
projects/app/src/pages/dataset/detail/components/Import/WebsiteConfig.tsx Normal file
View File

@ -0,0 +1,101 @@
import React from 'react';
import MyModal from '@/components/MyModal';
import { useTranslation } from 'next-i18next';
import { Box, Button, Input, ModalBody, ModalFooter } from '@chakra-ui/react';
import { strIsLink } from '@fastgpt/global/common/string/tools';
import { useToast } from '@/web/common/hooks/useToast';
import { useForm } from 'react-hook-form';
import { useConfirm } from '@/web/common/hooks/useConfirm';
type FormType = {
url?: string | undefined;
selector?: string | undefined;
};
const WebsiteConfigModal = ({
onClose,
onSuccess,
defaultValue = {
url: '',
selector: ''
}
}: {
onClose: () => void;
onSuccess: (data: FormType) => void;
defaultValue?: FormType;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const { register, handleSubmit } = useForm({
defaultValues: defaultValue
});
const isEdit = !!defaultValue.url;
const confirmTip = isEdit
? t('core.dataset.website.Confirm Update Tips')
: t('core.dataset.website.Confirm Create Tips');
const { ConfirmModal, openConfirm } = useConfirm({
type: 'common'
});
return (
<MyModal
isOpen
iconSrc="core/dataset/websiteDataset"
title={t('core.dataset.website.Config')}
onClose={onClose}
maxW={'500px'}
>
<ModalBody>
<Box fontSize={'sm'} color={'myGray.600'}>
{t('core.dataset.website.Config Description')}
</Box>
<Box mt={2}>
<Box>{t('core.dataset.website.Base Url')}</Box>
<Input
placeholder={t('core.dataset.collection.Website Link')}
{...register('url', {
required: true
})}
/>
</Box>
<Box mt={3}>
<Box>
{t('core.dataset.website.Selector')}({t('common.choosable')})
</Box>
<Input {...register('selector')} placeholder="body .content #document" />
</Box>
</ModalBody>
<ModalFooter>
<Button variant={'base'} onClick={onClose}>
{t('common.Close')}
</Button>
<Button
ml={2}
onClick={handleSubmit((data) => {
if (!data.url) return;
// check is link
if (!strIsLink(data.url)) {
return toast({
status: 'warning',
title: t('common.link.UnValid')
});
}
openConfirm(
() => {
onSuccess(data);
},
undefined,
confirmTip
)();
})}
>
{t('core.dataset.website.Start Sync')}
</Button>
</ModalFooter>
<ConfirmModal />
</MyModal>
);
};
export default WebsiteConfigModal;

93
projects/app/src/pages/dataset/detail/components/Info.tsx
View File

@ -7,7 +7,7 @@ import React, {
ForwardedRef
} from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, Button, FormControl, IconButton, Input } from '@chakra-ui/react';
import { Box, Flex, Button, FormControl, IconButton, Input, Textarea } from '@chakra-ui/react';
import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
import { delDatasetById } from '@/web/core/dataset/api';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
@ -18,21 +18,19 @@ import { UseFormReturn } from 'react-hook-form';
import { compressImgFileAndUpload } from '@/web/common/file/controller';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import Avatar from '@/components/Avatar';
import Tag from '@/components/Tag';
import MyTooltip from '@/components/MyTooltip';
import { useTranslation } from 'next-i18next';
import PermissionRadio from '@/components/support/permission/Radio';
import MySelect from '@/components/Select';
import { qaModelList } from '@/web/common/system/staticData';
export interface ComponentRef {
initInput: (tags: string) => void;
}
const Info = (
{ datasetId, form }: { datasetId: string; form: UseFormReturn<DatasetItemType, any> },
ref: ForwardedRef<ComponentRef>
) => {
const Info = ({
datasetId,
form
}: {
datasetId: string;
form: UseFormReturn<DatasetItemType, any>;
}) => {
const { t } = useTranslation();
const { getValues, formState, setValue, register, handleSubmit } = form;
const InputRef = useRef<HTMLInputElement>(null);
@ -52,7 +50,7 @@ const Info = (
multiple: false
});
const { datasetDetail, loadDatasetDetail, loadDatasets, updateDataset } = useDatasetStore();
const { datasetDetail, loadDatasets, updateDataset } = useDatasetStore();
/* 点击删除 */
const onclickDelKb = useCallback(async () => {
@ -121,8 +119,8 @@ const Info = (
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
@ -138,14 +136,6 @@ const Info = (
[setRefresh, setValue, toast]
);
useImperativeHandle(ref, () => ({
initInput: (tags: string) => {
if (InputRef.current) {
InputRef.current.value = tags;
}
}
}));
return (
<Box py={5} px={[5, 10]}>
<Flex mt={5} w={'100%'} alignItems={'center'}>
@ -154,18 +144,7 @@ const Info = (
</Box>
<Box flex={1}>{datasetDetail._id}</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
索引模型
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
MaxTokens
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
</Flex>
<Flex mt={5} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
知识库头像
@ -183,7 +162,7 @@ const Info = (
</MyTooltip>
</Box>
</Flex>
<FormControl mt={8} w={'100%'} display={'flex'} alignItems={'center'}>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
知识库名称
</Box>
@ -194,7 +173,19 @@ const Info = (
required: '知识库名称不能为空'
})}
/>
</FormControl>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
索引模型
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.Max Token')}
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
</Flex>
<Flex mt={6} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('dataset.Agent Model')}
@ -216,33 +207,9 @@ const Info = (
/>
</Box>
</Flex>
<Flex mt={8} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
标签
<MyTooltip label={'用空格隔开多个标签，便于搜索'} forceShow>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Input
flex={[1, '0 0 300px']}
ref={InputRef}
defaultValue={getValues('tags')}
placeholder={'标签,使用空格分割。'}
maxLength={30}
onChange={(e) => {
setValue('tags', e.target.value.split(' ').filter(Boolean));
setRefresh(!refresh);
}}
/>
<Flex w={'100%'} pl={['90px', '160px']} mt={2}>
{getValues('tags')
.filter(Boolean)
.map((item, i) => (
<Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
{item}
</Tag>
))}
</Flex>
<Flex mt={8} alignItems={'center'} w={'100%'}>
<Box flex={['0 0 90px', '0 0 160px']}>{t('common.Intro')}</Box>
<Textarea flex={[1, '0 0 300px']} {...register('intro')} placeholder={t('common.Intro')} />
</Flex>
{datasetDetail.isOwner && (
<Flex mt={5} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
@ -292,4 +259,4 @@ const Info = (
);
};
export default forwardRef(Info);
export default React.memo(Info);

12
projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
View File

@ -199,12 +199,12 @@ const InputDataModal = ({
<Box p={5} borderRight={theme.borders.base}>
<RawSourceText
w={'200px'}
className=""
className="textEllipsis3"
whiteSpace={'pre-wrap'}
sourceName={collection.sourceName}
sourceId={collection.sourceId}
mb={6}
fontSize={['14px', '16px']}
fontSize={'sm'}
/>
<SideTabs
list={tabList}
@ -220,14 +220,14 @@ const InputDataModal = ({
}}
/>
</Box>
<Flex flexDirection={'column'} px={5} py={3} flex={1} h={'100%'}>
<Box fontSize={'lg'} fontWeight={'bold'} mb={4}>
<Flex flexDirection={'column'} py={3} flex={1} h={'100%'}>
<Box fontSize={'lg'} px={5} fontWeight={'bold'} mb={4}>
{currentTab === TabEnum.content && (
<>{defaultValue.id ? t('dataset.data.Update Data') : t('dataset.data.Input Data')}</>
)}
{currentTab === TabEnum.index && <> {t('dataset.data.Index Edit')}</>}
</Box>
<Box flex={1} overflow={'auto'}>
<Box flex={1} px={5} overflow={'auto'}>
{currentTab === TabEnum.content && (
<>
<Box>
@ -358,7 +358,7 @@ const InputDataModal = ({
</Grid>
)}
</Box>
<Flex justifyContent={'flex-end'} mt={4}>
<Flex justifyContent={'flex-end'} px={5} mt={4}>
<Button variant={'base'} mr={3} isLoading={loading} onClick={onClose}>
{t('common.Close')}
</Button>

6
projects/app/src/pages/dataset/detail/components/Test.tsx
View File

@ -187,7 +187,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
</Box>
</Box>
{/* result show */}
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={1}>
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={'1 0 0'}>
{!datasetTestItem?.results || datasetTestItem.results.length === 0 ? (
<Flex
mt={[10, 0]}
@ -275,7 +275,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
<MyIcon name={'kbTest'} w={'14px'} />
<Progress
mx={2}
flex={1}
flex={'1 0 0'}
value={item.score * 100}
size="sm"
borderRadius={'20px'}
@ -283,7 +283,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
/>
<Box>{item.score.toFixed(4)}</Box>
</Flex>
<Box px={2} fontSize={'xs'} color={'myGray.600'}>
<Box px={2} fontSize={'xs'} color={'myGray.600'} wordBreak={'break-word'}>
<Box>{item.q}</Box>
<Box>{item.a}</Box>
</Box>

26
projects/app/src/pages/dataset/detail/index.tsx
View File

@ -7,7 +7,6 @@ import { useQuery } from '@tanstack/react-query';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { type ComponentRef } from './components/Info';
import Tabs from '@/components/Tabs';
import dynamic from 'next/dynamic';
import MyIcon from '@/components/Icon';
@ -25,6 +24,7 @@ import Script from 'next/script';
import CollectionCard from './components/CollectionCard';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useUserStore } from '@/web/support/user/useUserStore';
import { DatasetTypeMap } from '../../../../../../packages/global/core/dataset/constant';
const DataCard = dynamic(() => import('./components/DataCard'), {
ssr: false
@ -41,7 +41,6 @@ export enum TabEnum {
}
const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${TabEnum}` }) => {
const InfoRef = useRef<ComponentRef>(null);
const theme = useTheme();
const { t } = useTranslation();
const { toast } = useToast();
@ -77,7 +76,6 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
useQuery([datasetId], () => loadDatasetDetail(datasetId), {
onSuccess(res) {
form.reset(res);
InfoRef.current?.initInput(res.tags?.join(' '));
},
onError(err: any) {
router.replace(`/dataset/list`);
@ -107,14 +105,24 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
>
<Flex mb={4} alignItems={'center'}>
<Avatar src={datasetDetail.avatar} w={'34px'} borderRadius={'lg'} />
<Box ml={2} fontWeight={'bold'}>
{datasetDetail.name}
<Box ml={2}>
<Box fontWeight={'bold'}>{datasetDetail.name}</Box>
</Box>
</Flex>
{DatasetTypeMap[datasetDetail.type] && (
<Flex alignItems={'center'} pl={2}>
<MyIcon
name={DatasetTypeMap[datasetDetail.type]?.icon as any}
mr={1}
w={'16px'}
/>
<Box>{t(DatasetTypeMap[datasetDetail.type]?.label)}</Box>
</Flex>
)}
<SideTabs
flex={1}
mx={'auto'}
mt={2}
mt={3}
w={'100%'}
list={tabList}
activeId={currentTab}
@ -156,7 +164,7 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
borderRadius={'50%'}
aria-label={''}
/>
全部知识库
{t('core.dataset.All Dataset')}
</Flex>
</Flex>
) : (
@ -180,9 +188,7 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
{currentTab === TabEnum.collectionCard && <CollectionCard />}
{currentTab === TabEnum.dataCard && <DataCard />}
{currentTab === TabEnum.test && <Test datasetId={datasetId} />}
{currentTab === TabEnum.info && (
<Info ref={InfoRef} datasetId={datasetId} form={form} />
)}
{currentTab === TabEnum.info && <Info datasetId={datasetId} form={form} />}
</Box>
)}
</Flex>

148
projects/app/src/pages/dataset/list/component/CreateModal.tsx
View File

@ -1,14 +1,5 @@
import React, { useCallback, useState, useRef } from 'react';
import {
Box,
Flex,
Button,
ModalHeader,
ModalFooter,
ModalBody,
Input,
Image
} from '@chakra-ui/react';
import React, { useCallback, useState } from 'react';
import { Box, Flex, Button, ModalFooter, ModalBody, Input } from '@chakra-ui/react';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useForm } from 'react-hook-form';
import { compressImgFileAndUpload } from '@/web/common/file/controller';
@ -23,10 +14,11 @@ import MyModal from '@/components/MyModal';
import { postCreateDataset } from '@/web/core/dataset/api';
import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import MySelect from '@/components/Select';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { vectorModelList, qaModelList } from '@/web/common/system/staticData';
import Tag from '@/components/Tag';
import { useTranslation } from 'next-i18next';
import MyRadio from '@/components/common/MyRadio';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { feConfigs } from '@/web/common/system/staticData';
const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: string }) => {
const { t } = useTranslation();
@ -36,16 +28,15 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
const { isPc } = useSystemStore();
const { register, setValue, getValues, handleSubmit } = useForm<CreateDatasetParams>({
defaultValues: {
parentId,
type: DatasetTypeEnum.dataset,
avatar: '/icon/logo.svg',
name: '',
tags: '',
intro: '',
vectorModel: vectorModelList[0].model,
agentModel: qaModelList[0].model,
type: 'dataset',
parentId
agentModel: qaModelList[0].model
}
});
const InputRef = useRef<HTMLInputElement>(null);
const { File, onOpen: onOpenSelectFile } = useSelectFile({
fileType: '.jpg,.png',
@ -59,8 +50,8 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
setRefresh((state) => !state);
@ -97,32 +88,67 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
w={'450px'}
>
<ModalBody>
<Box color={'myGray.800'} fontWeight={'bold'}>
取个名字
</Box>
<Flex mt={3} alignItems={'center'}>
<MyTooltip label={'点击设置头像'}>
<Avatar
flexShrink={0}
src={getValues('avatar')}
w={['28px', '32px']}
h={['28px', '32px']}
cursor={'pointer'}
borderRadius={'md'}
onClick={onOpenSelectFile}
/>
</MyTooltip>
<Input
ml={3}
flex={1}
autoFocus
bg={'myWhite.600'}
maxLength={30}
{...register('name', {
required: '知识库名称不能为空~'
})}
<>
<Box mb={1} color={'myGray.800'} fontWeight={'bold'}>
{t('core.dataset.Dataset Type')}
</Box>
<MyRadio
gridGap={2}
gridTemplateColumns={'repeat(1,1fr)'}
list={[
{
title: t('core.dataset.Common Dataset'),
value: DatasetTypeEnum.dataset,
icon: 'core/dataset/commonDataset',
desc: t('core.dataset.Common Dataset Desc')
},
...(feConfigs.isPlus
? [
{
title: t('core.dataset.Website Dataset'),
value: DatasetTypeEnum.websiteDataset,
icon: 'core/dataset/websiteDataset',
desc: t('core.dataset.Website Dataset Desc')
}
]
: [])
]}
value={getValues('type')}
onChange={(e) => {
setValue('type', e as `${DatasetTypeEnum}`);
setRefresh(!refresh);
}}
/>
</Flex>
</>
<Box mt={5}>
<Box color={'myGray.800'} fontWeight={'bold'}>
取个名字
</Box>
<Flex mt={1} alignItems={'center'}>
<MyTooltip label={'点击设置头像'}>
<Avatar
flexShrink={0}
src={getValues('avatar')}
w={['28px', '32px']}
h={['28px', '32px']}
cursor={'pointer'}
borderRadius={'md'}
onClick={onOpenSelectFile}
/>
</MyTooltip>
<Input
ml={3}
flex={1}
autoFocus
bg={'myWhite.600'}
placeholder={t('common.Name')}
maxLength={30}
{...register('name', {
required: '知识库名称不能为空~'
})}
/>
</Flex>
</Box>
<Flex mt={6} alignItems={'center'}>
<Box flex={'0 0 100px'}>索引模型</Box>
<Box flex={1}>
@ -157,42 +183,14 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
/>
</Box>
</Flex>
<Flex mt={6} alignItems={'center'} w={'100%'}>
<Box flex={'0 0 100px'}>
标签
<MyTooltip label={'用空格隔开多个标签，便于搜索'} forceShow>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Input
flex={1}
ref={InputRef}
placeholder={'标签,使用空格分割。'}
maxLength={30}
onChange={(e) => {
setValue('tags', e.target.value);
setRefresh(!refresh);
}}
/>
</Flex>
<Flex mt={2} flexWrap={'wrap'}>
{getValues('tags')
.split(' ')
.filter(Boolean)
.map((item, i) => (
<Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
{item}
</Tag>
))}
</Flex>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={3} onClick={onClose}>
取消
{t('common.Close')}
</Button>
<Button isLoading={creating} onClick={handleSubmit((data) => onclickCreate(data))}>
确认创建
{t('common.Confirm Create')}
</Button>
</ModalFooter>

95
projects/app/src/pages/dataset/list/index.tsx
View File

@ -7,8 +7,7 @@ import {
useDisclosure,
Card,
MenuButton,
Image,
Link
Image
} from '@chakra-ui/react';
import { useRouter } from 'next/router';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
@ -28,8 +27,11 @@ import Avatar from '@/components/Avatar';
import MyIcon from '@/components/Icon';
import { serviceSideProps } from '@/web/common/utils/i18n';
import dynamic from 'next/dynamic';
import { FolderAvatarSrc, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import Tag from '@/components/Tag';
import {
FolderAvatarSrc,
DatasetTypeEnum,
DatasetTypeMap
} from '@fastgpt/global/core/dataset/constant';
import MyMenu from '@/components/MyMenu';
import { useRequest } from '@/web/common/hooks/useRequest';
import { useSystemStore } from '@/web/common/system/useSystemStore';
@ -55,12 +57,12 @@ const Kb = () => {
const DeleteTipsMap = useRef({
[DatasetTypeEnum.folder]: t('dataset.deleteFolderTips'),
[DatasetTypeEnum.dataset]: t('dataset.deleteDatasetTips')
[DatasetTypeEnum.dataset]: t('dataset.deleteDatasetTips'),
[DatasetTypeEnum.websiteDataset]: t('core.dataset.Delete Website Tips')
});
const { openConfirm, ConfirmModal } = useConfirm({
title: t('common.Delete Warning'),
content: ''
type: 'delete'
});
const { myDatasets, loadDatasets, setDatasets, updateDataset } = useDatasetStore();
const { onOpenModal: onOpenTitleModal, EditModal: EditTitleModal } = useEditTitle({
@ -110,14 +112,26 @@ const Kb = () => {
errorToast: t('dataset.Export Dataset Limit Error')
});
const { data, refetch } = useQuery(['loadDataset', parentId], () => {
const { data, refetch, isFetching } = useQuery(['loadDataset', parentId], () => {
return Promise.all([loadDatasets(parentId), getDatasetPaths(parentId)]);
});
const paths = data?.[1] || [];
const formatDatasets = useMemo(
() =>
myDatasets.map((item) => {
return {
...item,
label: DatasetTypeMap[item.type]?.label,
icon: DatasetTypeMap[item.type]?.icon
};
}),
[myDatasets]
);
return (
<PageContainer>
<PageContainer isLoading={isFetching}>
<Flex pt={3} px={5} alignItems={'center'}>
{/* url path */}
<ParentPaths
@ -179,7 +193,7 @@ const Kb = () => {
child: (
<Flex>
<Image src={'/imgs/module/db.png'} alt={''} w={'20px'} mr={1} />
{t('Dataset')}
{t('core.dataset.Dataset')}
</Flex>
),
onClick: onOpenCreateModal
@ -194,15 +208,15 @@ const Kb = () => {
gridGap={5}
userSelect={'none'}
>
{myDatasets.map((dataset) => (
{formatDatasets.map((dataset) => (
<Card
display={'flex'}
flexDirection={'column'}
key={dataset._id}
py={4}
py={3}
px={5}
cursor={'pointer'}
h={'130px'}
minH={'130px'}
border={theme.borders.md}
boxShadow={'none'}
position={'relative'}
@ -250,7 +264,7 @@ const Kb = () => {
parentId: dataset._id
}
});
} else if (dataset.type === DatasetTypeEnum.dataset) {
} else {
router.push({
pathname: '/dataset/detail',
query: {
@ -388,27 +402,22 @@ const Kb = () => {
{dataset.name}
</Box>
</Flex>
<Box flex={'1 0 0'} overflow={'hidden'} pt={2}>
<Flex>
{dataset.tags.filter(Boolean).map((tag, i) => (
<Tag key={i} mr={2} mb={2}>
{tag}
</Tag>
))}
</Flex>
<Box
flex={1}
className={'textEllipsis3'}
py={1}
wordBreak={'break-all'}
fontSize={'sm'}
color={'myGray.500'}
>
{dataset.intro || t('core.dataset.Intro Placeholder')}
</Box>
<Flex alignItems={'center'} fontSize={'sm'}>
<Box flex={1}>
<PermissionIconText permission={dataset.permission} color={'myGray.600'} />
</Box>
{dataset.type === DatasetTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box>
) : (
<>
<MyIcon mr={1} name="kbTest" w={'12px'} />
<Box color={'myGray.500'}>{dataset.vectorModel.name}</Box>
</>
)}
<MyIcon mr={1} name={dataset.icon as any} w={'12px'} />
<Box color={'myGray.500'}>{t(dataset.label)}</Box>
</Flex>
</Card>
))}
@ -417,7 +426,7 @@ const Kb = () => {
<Flex mt={'35vh'} flexDirection={'column'} alignItems={'center'}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
还没有知识库，快去创建一个吧！
{t('core.dataset.Empty Dataset Tips')}
</Box>
</Flex>
)}
@ -429,27 +438,19 @@ const Kb = () => {
onClose={() => setEditFolderData(undefined)}
editCallback={async (name) => {
try {
if (editFolderData.id) {
await putDatasetById({
id: editFolderData.id,
name
});
} else {
await postCreateDataset({
parentId,
name,
type: DatasetTypeEnum.folder,
avatar: FolderAvatarSrc,
tags: ''
});
}
await postCreateDataset({
parentId,
name,
type: DatasetTypeEnum.folder,
avatar: FolderAvatarSrc,
intro: ''
});
refetch();
} catch (error) {
return Promise.reject(error);
}
}}
isEdit={!!editFolderData.id}
name={editFolderData.name}
isEdit={false}
/>
)}
{!!moveDataId && (

4
projects/app/src/pages/plugin/list/component/EditModal.tsx
View File

@ -138,8 +138,8 @@ const CreateModal = ({
try {
const src = await compressImgFileAndUpload({
file,
maxW: 100,
maxH: 100
maxW: 300,
maxH: 300
});
setValue('avatar', src);
setRefresh((state) => !state);

8
projects/app/src/service/common/api/request.ts
View File

@ -96,18 +96,18 @@ export function request(url: string, data: any, config: ConfigType, method: Meth
* @param {Object} config
* @returns
*/
export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
export function GET<T = undefined>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
return request(url, params, config, 'GET');
}
export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
export function POST<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
return request(url, data, config, 'POST');
}
export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
export function PUT<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
return request(url, data, config, 'PUT');
}
export function DELETE<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
export function DELETE<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
return request(url, data, config, 'DELETE');
}

3
projects/app/src/service/core/ai/rerank.ts
View File

@ -20,7 +20,8 @@ export function reRankRecall({ query, inputs }: PostReRankProps) {
Authorization: `Bearer ${model.requestAuth}`
}
}
).finally(() => {
).then((data) => {
console.log('rerank time:', Date.now() - start);
return data;
});
}

29
projects/app/src/service/core/dataset/data/controller.ts
View File

@ -4,7 +4,8 @@ import {
PatchIndexesProps,
UpdateDatasetDataProps
} from '@fastgpt/global/core/dataset/controller';
import { deletePgDataById, insertData2Pg, updatePgDataById } from './pg';
import { deletePgDataById } from '@fastgpt/service/core/dataset/data/pg';
import { insertData2Pg, updatePgDataById } from './pg';
import { Types } from 'mongoose';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
@ -213,29 +214,3 @@ export async function updateData2Dataset({
tokenLen
};
}
/* delete all data by datasetIds */
export async function delDataByDatasetId({ datasetIds }: { datasetIds: string[] }) {
datasetIds = datasetIds.map((item) => String(item));
// delete pg data
await deletePgDataById(`dataset_id IN ('${datasetIds.join("','")}')`);
// delete dataset.datas
await MongoDatasetData.deleteMany({ datasetId: { $in: datasetIds } });
}
/**
* delete all data by collectionIds
*/
export async function delDataByCollectionId({ collectionIds }: { collectionIds: string[] }) {
const ids = collectionIds.map((item) => String(item));
// delete pg data
await deletePgDataById(`collection_id IN ('${ids.join("','")}')`);
// delete dataset.datas
await MongoDatasetData.deleteMany({ collectionId: { $in: ids } });
}
/**
* delete one data by mongoDataId
*/
export async function deleteDataByDataId(mongoDataId: string) {
await deletePgDataById(['data_id', mongoDataId]);
await MongoDatasetData.findByIdAndDelete(mongoDataId);
}

27
projects/app/src/service/core/dataset/data/pg.ts
View File

@ -5,7 +5,7 @@ import type {
} from '@fastgpt/global/core/dataset/type.d';
import { PgClient } from '@fastgpt/service/common/pg';
import { getVectorsByText } from '@/service/core/ai/vector';
import { delay } from '@/utils/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgSearchRawType } from '@fastgpt/global/core/dataset/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
@ -103,31 +103,6 @@ export async function updatePgDataById({
return updatePg();
}
export async function deletePgDataById(
where: ['id' | 'dataset_id' | 'collection_id' | 'data_id', string] | string
) {
let retry = 2;
async function deleteData(): Promise<any> {
try {
await PgClient.delete(PgDatasetTableName, {
where: [where]
});
} catch (error) {
if (--retry < 0) {
return Promise.reject(error);
}
await delay(500);
return deleteData();
}
}
await deleteData();
return {
tokenLen: 0
};
}
// ------------------ search start ------------------
type SearchProps = {
text: string;

Some files were not shown because too many files have changed in this diff Show More