diff --git a/packages/service/core/app/delete/index.ts b/packages/service/core/app/delete/index.ts index 465a17ef4..3f55b7881 100644 --- a/packages/service/core/app/delete/index.ts +++ b/packages/service/core/app/delete/index.ts @@ -31,10 +31,10 @@ export const addAppDeleteJob = (data: AppDeleteJobData) => { } }); - const jobId = `${data.teamId}:${data.appId}`; + const jobId = `${String(data.teamId)}:${String(data.appId)}`; // Use jobId to automatically prevent duplicate deletion tasks (BullMQ feature) - return appDeleteQueue.add('deleteapp', data, { + return appDeleteQueue.add('delete_app', data, { jobId, delay: 1000 // Delay 1 second to ensure API response completes }); diff --git a/packages/service/core/dataset/controller.ts b/packages/service/core/dataset/controller.ts index bab654958..4e3d9a369 100644 --- a/packages/service/core/dataset/controller.ts +++ b/packages/service/core/dataset/controller.ts @@ -68,7 +68,7 @@ export async function delDatasetRelevantData({ datasets, session }: { - datasets: DatasetSchemaType[]; + datasets: { _id: string; teamId: string }[]; session: ClientSession; }) { if (!datasets.length) return; @@ -115,24 +115,6 @@ export async function delDatasetRelevantData({ // Delete vector data await deleteDatasetDataVector({ teamId, datasetIds }); - for (const datasetId of datasetIds) { - // Delete dataset_data_texts in batches by datasetId - await MongoDatasetDataText.deleteMany({ - teamId, - datasetId - }).maxTimeMS(300000); // Reduce timeout for single batch - // Delete dataset_datas in batches by datasetId - await MongoDatasetData.deleteMany({ - teamId, - datasetId - }).maxTimeMS(300000); - } - - // Delete source: 兼容旧版的图片 - await delCollectionRelatedSource({ collections }); - // Delete vector data - await deleteDatasetDataVector({ teamId, datasetIds }); - // delete collections await MongoDatasetCollection.deleteMany({ teamId, diff --git a/packages/service/core/dataset/delete/index.ts b/packages/service/core/dataset/delete/index.ts index 6713ba329..189b158e0 100644 --- a/packages/service/core/dataset/delete/index.ts +++ b/packages/service/core/dataset/delete/index.ts @@ -31,11 +31,11 @@ export const addDatasetDeleteJob = (data: DatasetDeleteJobData) => { } }); - const jobId = `${data.teamId}:${data.datasetId}`; + const jobId = `${String(data.teamId)}:${String(data.datasetId)}`; // 使用去重机制,避免重复删除 - return datasetDeleteQueue.add(jobId, data, { - deduplication: { id: jobId }, + return datasetDeleteQueue.add('delete_dataset', data, { + jobId, delay: 1000 // 延迟1秒执行,确保API响应完成 }); }; diff --git a/packages/service/core/dataset/delete/processor.ts b/packages/service/core/dataset/delete/processor.ts index b0d24280c..154fc861f 100644 --- a/packages/service/core/dataset/delete/processor.ts +++ b/packages/service/core/dataset/delete/processor.ts @@ -1,8 +1,7 @@ import type { Processor } from 'bullmq'; -import type { DatasetDeleteJobData } from './index'; +import { addDatasetDeleteJob, type DatasetDeleteJobData } from './index'; import { delDatasetRelevantData, findDatasetAndAllChildren } from '../controller'; import { addLog } from '../../../common/system/log'; -import type { DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetCollectionTags } from '../tag/schema'; import { removeDatasetSyncJobScheduler } from '../datasetSync'; import { mongoSessionRun } from '../../../common/mongo/sessionRun'; @@ -12,36 +11,53 @@ import { MongoDatasetTraining } from '../training/schema'; export const deleteDatasetsImmediate = async ({ teamId, - datasets + datasetIds }: { teamId: string; - datasets: DatasetSchemaType[]; + datasetIds: string[]; }) => { - const datasetIds = datasets.map((d) => d._id); - // delete training data - MongoDatasetTraining.deleteMany({ + await MongoDatasetTraining.deleteMany({ teamId, datasetId: { $in: datasetIds } }); // Remove cron job + await Promise.all( + datasetIds.map((id) => { + return removeDatasetSyncJobScheduler(id); + }) + ); +}; +// Clear a team datasets +export const deleteTeamAllDatasets = async (teamId: string) => { + const datasets = await MongoDataset.find( + { + teamId + }, + { _id: 1, parentId: 1 } + ); + await deleteDatasetsImmediate({ + teamId, + datasetIds: datasets.map((d) => d._id) + }); await Promise.all( datasets.map((dataset) => { - // 只处理已标记删除的数据集 - if (datasetIds.includes(dataset._id)) { - return removeDatasetSyncJobScheduler(dataset._id); - } + if (dataset.parentId) return; + return addDatasetDeleteJob({ + teamId, + datasetId: dataset._id + }); }) ); }; -export const deleteDatasets = async ({ +const deleteDatasets = async ({ teamId, datasets }: { teamId: string; - datasets: DatasetSchemaType[]; + datasets: { _id: string; avatar: string; teamId: string }[]; }) => { const datasetIds = datasets.map((d) => d._id); @@ -81,7 +97,8 @@ export const datasetDeleteProcessor: Processor = async (jo // 1. 查找知识库及其所有子知识库 const datasets = await findDatasetAndAllChildren({ teamId, - datasetId + datasetId, + fields: '_id teamId avatar' }); if (!datasets || datasets.length === 0) { diff --git a/packages/service/support/activity/promotion/schema.ts b/packages/service/support/activity/promotion/schema.ts index aee481480..6798bdc10 100644 --- a/packages/service/support/activity/promotion/schema.ts +++ b/packages/service/support/activity/promotion/schema.ts @@ -29,6 +29,8 @@ const PromotionRecordSchema = new Schema({ } }); +PromotionRecordSchema.index({ userId: 1 }); + export const MongoPromotionRecord = getMongoModel( 'promotionRecord', PromotionRecordSchema diff --git a/projects/app/src/pages/api/core/dataset/delete.ts b/projects/app/src/pages/api/core/dataset/delete.ts index 94011b056..b94752b8c 100644 --- a/projects/app/src/pages/api/core/dataset/delete.ts +++ b/projects/app/src/pages/api/core/dataset/delete.ts @@ -32,14 +32,16 @@ async function handler(req: NextApiRequest) { const deleteDatasets = await findDatasetAndAllChildren({ teamId, - datasetId + datasetId, + fields: '_id' }); + const datasetIds = deleteDatasets.map((d) => d._id); await mongoSessionRun(async (session) => { // 1. Mark as deleted await MongoDataset.updateMany( { - _id: deleteDatasets.map((d) => d._id), + _id: datasetIds, teamId }, { @@ -52,7 +54,7 @@ async function handler(req: NextApiRequest) { await deleteDatasetsImmediate({ teamId, - datasets: deleteDatasets + datasetIds }); // 2. Add to delete queue diff --git a/projects/app/src/service/common/bullmq/index.ts b/projects/app/src/service/common/bullmq/index.ts index 26498e8ed..67990fcec 100644 --- a/projects/app/src/service/common/bullmq/index.ts +++ b/projects/app/src/service/common/bullmq/index.ts @@ -5,7 +5,5 @@ import { initAppDeleteWorker } from '@fastgpt/service/core/app/delete'; export const initBullMQWorkers = () => { addLog.info('Init BullMQ Workers...'); - initS3MQWorker(); - initDatasetDeleteWorker(); - initAppDeleteWorker(); + return Promise.all([initS3MQWorker(), initDatasetDeleteWorker(), initAppDeleteWorker()]); }; diff --git a/projects/app/test/api/core/app/delete.test.ts b/projects/app/test/api/core/app/delete.test.ts index d5a44b411..655693067 100644 --- a/projects/app/test/api/core/app/delete.test.ts +++ b/projects/app/test/api/core/app/delete.test.ts @@ -84,7 +84,7 @@ describe('App Delete Queue', () => { } }); - expect(mockQueue.add).toHaveBeenCalledWith('deleteapp', jobData, { + expect(mockQueue.add).toHaveBeenCalledWith('delete_app', jobData, { jobId: 'team-123:app-123', delay: 1000 }); @@ -106,7 +106,7 @@ describe('App Delete Queue', () => { await addAppDeleteJob(jobData); expect(mockQueue.add).toHaveBeenCalledWith( - 'deleteapp', + 'delete_app', jobData, expect.objectContaining({ jobId: 'team-xyz:app-abc' @@ -155,7 +155,7 @@ describe('App Delete API Integration', () => { // Verify queue job was added expect(mockQueue.add).toHaveBeenCalledWith( - 'deleteapp', + 'delete_app', { teamId: rootUser.teamId, appId: String(testApp._id)