mirror of
https://github.com/labring/FastGPT.git
synced 2025-12-25 20:02:47 +00:00
perf: init shell
This commit is contained in:
parent
d8cc4a0d52
commit
8544e14110
|
|
@ -88,6 +88,11 @@ export type OutLinkSchema<T extends OutlinkAppType = undefined> = {
|
||||||
};
|
};
|
||||||
|
|
||||||
app: T;
|
app: T;
|
||||||
|
|
||||||
|
//@deprecated
|
||||||
|
responseDetail?: boolean;
|
||||||
|
showNodeStatus?: boolean;
|
||||||
|
showRawSource?: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Edit the Outlink
|
// Edit the Outlink
|
||||||
|
|
|
||||||
|
|
@ -85,7 +85,12 @@ const OutLinkSchema = new Schema({
|
||||||
},
|
},
|
||||||
defaultResponse: {
|
defaultResponse: {
|
||||||
type: String
|
type: String
|
||||||
}
|
},
|
||||||
|
|
||||||
|
//@deprecated
|
||||||
|
responseDetail: Boolean,
|
||||||
|
showNodeStatus: Boolean,
|
||||||
|
showRawSource: Boolean
|
||||||
});
|
});
|
||||||
|
|
||||||
OutLinkSchema.virtual('associatedApp', {
|
OutLinkSchema.virtual('associatedApp', {
|
||||||
|
|
|
||||||
|
|
@ -105,7 +105,7 @@ const PlaygroundVisibilityConfig = ({ appId }: { appId: string }) => {
|
||||||
{t('publish:private_config')}
|
{t('publish:private_config')}
|
||||||
</Box>
|
</Box>
|
||||||
|
|
||||||
<Grid templateColumns={'1fr 1fr'} gap={4} mt={4}>
|
<Grid templateColumns={'1fr 1fr'} gap={4} mt={4} w={'400px'}>
|
||||||
<Flex alignItems={'center'}>
|
<Flex alignItems={'center'}>
|
||||||
<FormLabel fontSize={'12px'} flex={'0 0 127px'}>
|
<FormLabel fontSize={'12px'} flex={'0 0 127px'}>
|
||||||
{t('publish:show_node')}
|
{t('publish:show_node')}
|
||||||
|
|
@ -136,9 +136,6 @@ const PlaygroundVisibilityConfig = ({ appId }: { appId: string }) => {
|
||||||
isChecked={showCite}
|
isChecked={showCite}
|
||||||
/>
|
/>
|
||||||
</Flex>
|
</Flex>
|
||||||
</Grid>
|
|
||||||
|
|
||||||
<Grid templateColumns={'1fr 1fr'} gap={4} mt={4}>
|
|
||||||
<Flex>
|
<Flex>
|
||||||
<Flex alignItems={'center'} flex={'0 0 127px'}>
|
<Flex alignItems={'center'} flex={'0 0 127px'}>
|
||||||
<FormLabel fontSize={'12px'}>{t('common:core.app.share.Show full text')}</FormLabel>
|
<FormLabel fontSize={'12px'}>{t('common:core.app.share.Show full text')}</FormLabel>
|
||||||
|
|
|
||||||
|
|
@ -1,17 +1,12 @@
|
||||||
import { NextAPI } from '@/service/middleware/entry';
|
import { NextAPI } from '@/service/middleware/entry';
|
||||||
import { batchRun } from '@fastgpt/global/common/system/utils';
|
|
||||||
import { getQueue, QueueNames } from '@fastgpt/service/common/bullmq';
|
|
||||||
import type { S3MQJobData } from '@fastgpt/service/common/s3/mq';
|
|
||||||
import { addLog } from '@fastgpt/service/common/system/log';
|
|
||||||
import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
|
import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
|
||||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||||
import { PublishChannelEnum } from '@fastgpt/global/support/outLink/constant';
|
import { PublishChannelEnum } from '@fastgpt/global/support/outLink/constant';
|
||||||
import { connectionMongo } from '@fastgpt/service/common/mongo';
|
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
|
||||||
|
import { addLog } from '@fastgpt/service/common/system/log';
|
||||||
|
|
||||||
export type ResponseType = {
|
export type ResponseType = {
|
||||||
message: string;
|
message: string;
|
||||||
retriedCount: number;
|
|
||||||
failedCount: number;
|
|
||||||
shareLinkMigration: {
|
shareLinkMigration: {
|
||||||
totalRecords: number;
|
totalRecords: number;
|
||||||
updatedRecords: number;
|
updatedRecords: number;
|
||||||
|
|
@ -32,57 +27,6 @@ export type ResponseType = {
|
||||||
* - showRawSource -> canDownloadSource
|
* - showRawSource -> canDownloadSource
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
|
||||||
* 功能1: 重试所有失败的 S3 删除任务
|
|
||||||
*/
|
|
||||||
async function retryFailedS3DeleteJobs(): Promise<{
|
|
||||||
retriedCount: number;
|
|
||||||
failedCount: number;
|
|
||||||
}> {
|
|
||||||
const queue = getQueue<S3MQJobData>(QueueNames.s3FileDelete);
|
|
||||||
const failedJobs = await queue.getFailed();
|
|
||||||
console.log(`Found ${failedJobs.length} failed S3 delete jobs`);
|
|
||||||
|
|
||||||
let retriedCount = 0;
|
|
||||||
|
|
||||||
await batchRun(
|
|
||||||
failedJobs,
|
|
||||||
async (job) => {
|
|
||||||
addLog.debug(`Retrying S3 delete job with new attempts`, { retriedCount });
|
|
||||||
try {
|
|
||||||
// Remove old job and recreate with new attempts
|
|
||||||
const jobData = job.data;
|
|
||||||
await job.remove();
|
|
||||||
|
|
||||||
// Add new job with more attempts
|
|
||||||
await queue.add('delete-s3-files', jobData, {
|
|
||||||
attempts: 10,
|
|
||||||
removeOnFail: {
|
|
||||||
count: 10000, // 保留10000个失败任务
|
|
||||||
age: 14 * 24 * 60 * 60 // 14 days
|
|
||||||
},
|
|
||||||
removeOnComplete: true,
|
|
||||||
backoff: {
|
|
||||||
delay: 2000,
|
|
||||||
type: 'exponential'
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
retriedCount++;
|
|
||||||
console.log(`Retried S3 delete job ${job.id} with new attempts`);
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Failed to retry S3 delete job ${job.id}:`, error);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
100
|
|
||||||
);
|
|
||||||
|
|
||||||
return {
|
|
||||||
retriedCount,
|
|
||||||
failedCount: failedJobs.length
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 功能2和3: 处理 OutLink 记录的数据迁移
|
* 功能2和3: 处理 OutLink 记录的数据迁移
|
||||||
* - 添加 showFullText 字段
|
* - 添加 showFullText 字段
|
||||||
|
|
@ -102,143 +46,22 @@ async function migrateOutLinkData(): Promise<{
|
||||||
updated: number;
|
updated: number;
|
||||||
}> = [];
|
}> = [];
|
||||||
|
|
||||||
// 获取 MongoDB 原生集合,绕过 Mongoose 的严格模式
|
|
||||||
const db = connectionMongo.connection.db;
|
|
||||||
if (!db) {
|
|
||||||
throw new Error('Database connection not established');
|
|
||||||
}
|
|
||||||
const outLinkCollection = db.collection('outlinks');
|
|
||||||
|
|
||||||
// 1. 为所有 share 类型的记录添加 showFullText 字段
|
// 1. 为所有 share 类型的记录添加 showFullText 字段
|
||||||
const shareLinks = await outLinkCollection
|
const shareLinks = await MongoOutLink.find({
|
||||||
.find({
|
type: PublishChannelEnum.share
|
||||||
type: PublishChannelEnum.share,
|
});
|
||||||
showFullText: { $exists: false } // 只查找没有 showFullText 字段的记录
|
|
||||||
})
|
|
||||||
.toArray();
|
|
||||||
|
|
||||||
if (shareLinks.length > 0) {
|
for await (const link of shareLinks) {
|
||||||
// 批量更新添加 showFullText 字段
|
try {
|
||||||
const showFullTextOps = shareLinks.map((link: any) => ({
|
link.showFullText = link.showFullText ?? link.showRawSource ?? true;
|
||||||
updateOne: {
|
link.showRunningStatus = link.showRunningStatus ?? link.showNodeStatus ?? false;
|
||||||
filter: { _id: link._id },
|
link.showCite = link.showCite ?? link.responseDetail ?? false;
|
||||||
update: { $set: { showFullText: link.showRawSource ?? true } }
|
link.canDownloadSource = link.canDownloadSource ?? link.showRawSource ?? false;
|
||||||
}
|
await link.save();
|
||||||
}));
|
addLog.info(`[initv4145] 迁移 OutLink 数据成功: ${link.shareId}`);
|
||||||
|
} catch (error) {
|
||||||
const showFullTextResult = await outLinkCollection.bulkWrite(showFullTextOps);
|
addLog.error('[initv4145] 迁移 OutLink 数据失败:', error);
|
||||||
totalUpdated += showFullTextResult.modifiedCount;
|
}
|
||||||
updateResults.push({
|
|
||||||
operation: 'Add showFullText field',
|
|
||||||
updated: showFullTextResult.modifiedCount
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(`Added showFullText field to ${showFullTextResult.modifiedCount} share links`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. 重命名字段:showNodeStatus -> showRunningStatus
|
|
||||||
const showNodeStatusLinks = await outLinkCollection
|
|
||||||
.find({
|
|
||||||
showNodeStatus: { $exists: true },
|
|
||||||
showRunningStatus: { $exists: false }
|
|
||||||
})
|
|
||||||
.toArray();
|
|
||||||
|
|
||||||
if (showNodeStatusLinks.length > 0) {
|
|
||||||
const renameNodeStatusOps = showNodeStatusLinks.map((link: any) => ({
|
|
||||||
updateOne: {
|
|
||||||
filter: { _id: link._id },
|
|
||||||
update: [
|
|
||||||
{
|
|
||||||
$set: { showRunningStatus: '$showNodeStatus' }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
$unset: 'showNodeStatus'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
const renameNodeStatusResult = await outLinkCollection.bulkWrite(renameNodeStatusOps);
|
|
||||||
totalUpdated += renameNodeStatusResult.modifiedCount;
|
|
||||||
updateResults.push({
|
|
||||||
operation: 'Rename showNodeStatus to showRunningStatus',
|
|
||||||
updated: renameNodeStatusResult.modifiedCount
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
`Renamed showNodeStatus to showRunningStatus for ${renameNodeStatusResult.modifiedCount} links`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. 重命名字段:responseDetail -> showCite
|
|
||||||
const responseDetailLinks = await outLinkCollection
|
|
||||||
.find({
|
|
||||||
responseDetail: { $exists: true },
|
|
||||||
showCite: { $exists: false }
|
|
||||||
})
|
|
||||||
.toArray();
|
|
||||||
|
|
||||||
if (responseDetailLinks.length > 0) {
|
|
||||||
const renameResponseDetailOps = responseDetailLinks.map((link: any) => ({
|
|
||||||
updateOne: {
|
|
||||||
filter: { _id: link._id },
|
|
||||||
update: [
|
|
||||||
{
|
|
||||||
$set: { showCite: '$responseDetail' }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
$unset: 'responseDetail'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
const renameResponseDetailResult = await outLinkCollection.bulkWrite(renameResponseDetailOps);
|
|
||||||
totalUpdated += renameResponseDetailResult.modifiedCount;
|
|
||||||
updateResults.push({
|
|
||||||
operation: 'Rename responseDetail to showCite',
|
|
||||||
updated: renameResponseDetailResult.modifiedCount
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
`Renamed responseDetail to showCite for ${renameResponseDetailResult.modifiedCount} links`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. 重命名字段:showRawSource -> canDownloadSource
|
|
||||||
const showRawSourceLinks = await outLinkCollection
|
|
||||||
.find({
|
|
||||||
showRawSource: { $exists: true },
|
|
||||||
canDownloadSource: { $exists: false }
|
|
||||||
})
|
|
||||||
.toArray();
|
|
||||||
|
|
||||||
if (showRawSourceLinks.length > 0) {
|
|
||||||
const renameRawSourceOps = showRawSourceLinks.map((link: any) => ({
|
|
||||||
updateOne: {
|
|
||||||
filter: { _id: link._id },
|
|
||||||
update: [
|
|
||||||
{
|
|
||||||
$set: { canDownloadSource: '$showRawSource' }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
$unset: 'showRawSource'
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
const renameRawSourceResult = await outLinkCollection.bulkWrite(renameRawSourceOps);
|
|
||||||
totalUpdated += renameRawSourceResult.modifiedCount;
|
|
||||||
updateResults.push({
|
|
||||||
operation: 'Rename showRawSource to canDownloadSource',
|
|
||||||
updated: renameRawSourceResult.modifiedCount
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
`Renamed showRawSource to canDownloadSource for ${renameRawSourceResult.modifiedCount} links`
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|
@ -257,9 +80,6 @@ async function handler(
|
||||||
): Promise<ResponseType> {
|
): Promise<ResponseType> {
|
||||||
await authCert({ req, authRoot: true });
|
await authCert({ req, authRoot: true });
|
||||||
|
|
||||||
// 执行功能1: 重试 S3 删除任务
|
|
||||||
const s3JobResult = await retryFailedS3DeleteJobs();
|
|
||||||
|
|
||||||
// 执行功能2&3: OutLink 数据迁移
|
// 执行功能2&3: OutLink 数据迁移
|
||||||
let shareLinkMigration = {
|
let shareLinkMigration = {
|
||||||
totalRecords: 0,
|
totalRecords: 0,
|
||||||
|
|
@ -276,8 +96,6 @@ async function handler(
|
||||||
|
|
||||||
return {
|
return {
|
||||||
message: `Completed v4.14.5 initialization: S3 job retries and outLink migration`,
|
message: `Completed v4.14.5 initialization: S3 job retries and outLink migration`,
|
||||||
retriedCount: s3JobResult.retriedCount,
|
|
||||||
failedCount: s3JobResult.failedCount,
|
|
||||||
shareLinkMigration
|
shareLinkMigration
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue