FastGPT/projects/app/src/pages/api/common/file/previewContent.ts
Archer c6d9b15897
Some checks are pending
Build FastGPT images in Personal warehouse / build-fastgpt-images (push) Waiting to run
External dataset (#1497)
* perf: read rawText and chunk code

* perf: read raw text

* perf: read rawtext

* perf: token count

* log
2024-05-16 11:47:53 +08:00

51 lines
1.5 KiB
TypeScript

/*
Read db file content and response 3000 words
*/
import type { NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authFile } from '@fastgpt/service/support/permission/auth/file';
import { NextAPI } from '@/service/middle/entry';
import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { readDatasetSourceRawText } from '@fastgpt/service/core/dataset/read';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
export type PreviewContextProps = {
type: DatasetSourceReadTypeEnum;
sourceId: string;
isQAImport?: boolean;
selector?: string;
};
async function handler(req: ApiRequestProps<PreviewContextProps>, res: NextApiResponse<any>) {
const { type, sourceId, isQAImport, selector } = req.body;
if (!sourceId) {
throw new Error('fileId is empty');
}
const { teamId } = await (async () => {
if (type === DatasetSourceReadTypeEnum.fileLocal) {
return authFile({ req, authToken: true, authApiKey: true, fileId: sourceId });
}
return authCert({ req, authApiKey: true, authToken: true });
})();
const rawText = await readDatasetSourceRawText({
teamId,
type,
sourceId: sourceId,
isQAImport,
selector
});
jsonRes(res, {
data: {
previewContent: rawText.slice(0, 3000),
totalLength: rawText.length
}
});
}
export default NextAPI(handler);