perf: s3 upload by buffer

packages/service/common/s3/buckets/base.ts

@@ -10,11 +10,11 @@ import {
 import { defaultS3Options, getSystemMaxFileSize, Mimes } from '../constants';
 import path from 'node:path';
 import { MongoS3TTL } from '../schema';
-import { getNanoid } from '@fastgpt/global/common/string/tools';
 import { addHours, addMinutes } from 'date-fns';
 import { addLog } from '../../system/log';
 import { addS3DelJob } from '../mq';
 import { type Readable } from 'node:stream';
+import { type UploadFileByBufferParams, UploadFileByBufferSchema } from '../type';
 
 export class S3BaseBucket {
   private _client: Client;
@@ -251,4 +251,25 @@ export class S3BaseBucket {
 
     return await this.client.presignedGetObject(this.name, key, expires);
   }
+
+  async uploadFileByBuffer(params: UploadFileByBufferParams) {
+    const { key, buffer, contentType } = UploadFileByBufferSchema.parse(params);
+
+    await MongoS3TTL.create({
+      minioKey: key,
+      bucketName: this.name,
+      expiredTime: addHours(new Date(), 1)
+    });
+    await this.putObject(key, buffer, undefined, {
+      'Content-Type': contentType || 'application/octet-stream'
+    });
+
+    return {
+      key,
+      accessUrl: await this.createPreviewUrl({
+        key,
+        expiredHours: 2
+      })
+    };
+  }
 }

packages/service/common/s3/sources/chat/index.ts

@@ -1,13 +1,13 @@
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { S3PrivateBucket } from '../../buckets/private';
 import { S3Sources } from '../../type';
-import type { UploadFileParams } from './type';
 import {
   type CheckChatFileKeys,
   type DelChatFileByPrefixParams,
   ChatFileUploadSchema,
   DelChatFileByPrefixSchema,
-  UploadChatFileSchema
+  UploadChatFileSchema,
+  type UploadFileParams
 } from './type';
 import { differenceInHours } from 'date-fns';
 import { S3Buckets } from '../../constants';
@@ -97,7 +97,7 @@ export class S3ChatSource {
     const { fileKey } = getFileS3Key.chat({ appId, chatId, uId, filename });
     return await this.bucket.createPostPresignedUrl(
       { rawKey: fileKey, filename },
-      { expiredHours: expiredTime ? differenceInHours(new Date(), expiredTime) : 24 }
+      { expiredHours: expiredTime ? differenceInHours(expiredTime, new Date()) : 24 }
     );
   }
 
@@ -112,8 +112,8 @@ export class S3ChatSource {
     return this.bucket.addDeleteJob({ key });
   }
 
-  async uploadFileByBuffer(params: UploadFileParams) {
-    const { appId, chatId, uId, filename, expiredTime, buffer, contentType } =
+  async uploadChatFileByBuffer(params: UploadFileParams) {
+    const { appId, chatId, uId, filename, buffer, contentType } =
       UploadChatFileSchema.parse(params);
     const { fileKey } = getFileS3Key.chat({
       appId,
@@ -122,17 +122,11 @@ export class S3ChatSource {
       filename
     });
 
-    await this.bucket.putObject(fileKey, buffer, undefined, {
-      'Content-Type': contentType || 'application/octet-stream'
+    return this.bucket.uploadFileByBuffer({
+      key: fileKey,
+      buffer,
+      contentType
     });
-
-    return {
-      fileKey,
-      accessUrl: await this.bucket.createPreviewUrl({
-        key: fileKey,
-        expiredHours: expiredTime ? differenceInHours(new Date(), expiredTime) : 24
-      })
-    };
   }
 }
 

packages/service/common/s3/sources/chat/type.ts

@@ -22,7 +22,6 @@ export const UploadChatFileSchema = z.object({
   chatId: z.string().nonempty(),
   uId: z.string().nonempty(),
   filename: z.string().nonempty(),
-  expiredTime: z.date().optional(),
   buffer: z.instanceof(Buffer),
   contentType: z.string().optional()
 });

packages/service/common/s3/type.ts

@@ -56,6 +56,13 @@ export const UploadImage2S3BucketParamsSchema = z.object({
 });
 export type UploadImage2S3BucketParams = z.infer<typeof UploadImage2S3BucketParamsSchema>;
 
+export const UploadFileByBufferSchema = z.object({
+  buffer: z.instanceof(Buffer),
+  contentType: z.string().optional(),
+  key: z.string().nonempty()
+});
+export type UploadFileByBufferParams = z.infer<typeof UploadFileByBufferSchema>;
+
 declare global {
   var s3BucketMap: {
     [key: string]: S3BaseBucket;

packages/service/core/dataset/controller.ts

@@ -96,53 +96,13 @@ export async function delDatasetRelevantData({
     datasetId: { $in: datasetIds }
   });
 
-  // Delete dataset_data_texts in batches by datasetId
   for (const datasetId of datasetIds) {
+    // Delete dataset_data_texts in batches by datasetId
     await MongoDatasetDataText.deleteMany({
       teamId,
       datasetId
     }).maxTimeMS(300000); // Reduce timeout for single batch
-  }
-  // Delete dataset_datas in batches by datasetId
-  for (const datasetId of datasetIds) {
-    await MongoDatasetData.deleteMany({
-      teamId,
-      datasetId
-    }).maxTimeMS(300000);
-  }
-
-  await delCollectionRelatedSource({ collections });
-  // Delete vector data
-  await deleteDatasetDataVector({ teamId, datasetIds });
-
-  // Delete dataset_data_texts in batches by datasetId
-  for (const datasetId of datasetIds) {
-    await MongoDatasetDataText.deleteMany({
-      teamId,
-      datasetId
-    }).maxTimeMS(300000); // Reduce timeout for single batch
-  }
-  // Delete dataset_datas in batches by datasetId
-  for (const datasetId of datasetIds) {
-    await MongoDatasetData.deleteMany({
-      teamId,
-      datasetId
-    }).maxTimeMS(300000);
-  }
-
-  await delCollectionRelatedSource({ collections });
-  // Delete vector data
-  await deleteDatasetDataVector({ teamId, datasetIds });
-
-  // Delete dataset_data_texts in batches by datasetId
-  for (const datasetId of datasetIds) {
-    await MongoDatasetDataText.deleteMany({
-      teamId,
-      datasetId
-    }).maxTimeMS(300000); // Reduce timeout for single batch
-  }
-  // Delete dataset_datas in batches by datasetId
-  for (const datasetId of datasetIds) {
+    // Delete dataset_datas in batches by datasetId
     await MongoDatasetData.deleteMany({
       teamId,
       datasetId

packages/web/i18n/en/account.json

@@ -11,7 +11,6 @@
   "create_model": "Add new model",
   "custom_domain": "Custom Domain",
   "custom_model": "custom model",
-  "custom_domain": "custom domain",
   "custom_domain.domain": "Domain",
   "custom_domain.provider": "Domain Provider",
   "custom_domain.registration_hint": "Please prepare your own domain and complete the ICP filing through <bold>{{provider}}</bold>, then fill in the domain in the input box below",

packages/web/i18n/zh-CN/account.json

@@ -16,7 +16,6 @@
   "custom_config_details": "定制配置详情",
   "custom_domain": "自定义域名",
   "custom_model": "自定义模型",
-  "custom_domain": "自定义域名",
   "custom_domain.domain": "域名",
   "custom_domain.provider": "域名备案商",
   "custom_domain.registration_hint": "请自备域名并通过 <bold>{{provider}}</bold> 完成备案后，将域名填入下方输入框中",

packages/web/i18n/zh-Hant/account.json

@@ -14,7 +14,6 @@
   "create_channel": "新增管道",
   "create_model": "新增模型",
   "custom_config_details": "定制配置詳情",
-  "custom_domain": "自定義域名",
   "custom_model": "自訂模型",
   "custom_domain": "自訂域名",
   "custom_domain.domain": "域名",

projects/app/src/pageComponents/app/detail/Publish/Wecom/WecomEditModal.tsx

@@ -15,6 +15,7 @@ import { useMyStep } from '@fastgpt/web/hooks/useStep';
 import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import { format } from 'date-fns';
 import { ShareLinkContainer } from '../components/showShareLinkModal';
+import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
 
 const WecomEditModal = ({
   appId,
@@ -169,7 +170,7 @@ const WecomEditModal = ({
                     type="datetime-local"
                     defaultValue={
                       defaultData.limit?.expiredTime
-                        ? format(defaultData.limit?.expiredTime, 'YYYY-MM-DDTHH:mm')
+                        ? formatTime2YMDHM(defaultData.limit?.expiredTime)
                         : ''
                     }
                     onChange={(e) => {