15308555518/ FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-12-25 20:02:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

chat file url white list (#6053) 

* chat file url white list

* perf: white list

---------

Co-authored-by: archer <545436317@qq.com>
This commit is contained in:
heheer 2025-12-08 17:10:23 +08:00 committed by GitHub
parent 2ccb5b50c6
commit 5aaf123135
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 106 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
document/content/docs/upgrading/4-14/4144.mdx
View File

@ -29,6 +29,7 @@ curl --location --request POST 'https://{{host}}/api/admin/initv4144' \
3. 对话日志支持展示 IP 地址归属地。
4. 通过 API 上传本地文件至知识库，保存至 S3。同时将旧版 Gridfs 代码全部移除。
5. 新版订阅套餐逻辑。
6. 支持配置对话文件白名单。
## ⚙️ 优化
@ -36,6 +37,7 @@ curl --location --request POST 'https://{{host}}/api/admin/initv4144' \
2. 问题优化采用 JinaAI 的边际收益公式，获取最大边际收益的检索词。
3. 用户通知，支持中英文，以及优化模板。
4. 删除知识库采用队列异步删除模式。
5. LLM 请求时，图片无效报错提示。
## 🐛 修复

2
document/data/doc-last-modified.json
View File

@ -118,7 +118,7 @@
"document/content/docs/upgrading/4-14/4141.mdx": "2025-11-19T10:15:27+08:00",
"document/content/docs/upgrading/4-14/4142.mdx": "2025-11-18T19:27:14+08:00",
"document/content/docs/upgrading/4-14/4143.mdx": "2025-11-26T20:52:05+08:00",
"document/content/docs/upgrading/4-14/4144.mdx": "2025-12-07T14:24:15+08:00",
"document/content/docs/upgrading/4-14/4144.mdx": "2025-12-08T01:44:15+08:00",
"document/content/docs/upgrading/4-8/40.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-8/41.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-8/42.mdx": "2025-08-02T19:38:37+08:00",

1
packages/global/common/system/types/index.d.ts vendored
View File

@ -146,6 +146,7 @@ export type SystemEnvType = {
chatApiKey?: string;
customPdfParse?: customPdfParseType;
fileUrlWhitelist?: string[];
};
export type customPdfParseType = {

51
packages/service/common/security/fileUrlValidator.ts Normal file
View File

@ -0,0 +1,51 @@
const systemWhiteList = (() => {
const list: string[] = [];
if (process.env.S3_ENDPOINT) {
list.push(process.env.S3_ENDPOINT);
}
if (process.env.S3_EXTERNAL_BASE_URL) {
try {
const urlData = new URL(process.env.S3_EXTERNAL_BASE_URL);
list.push(urlData.hostname);
} catch (error) {}
}
if (process.env.FE_DOMAIN) {
try {
const urlData = new URL(process.env.FE_DOMAIN);
list.push(urlData.hostname);
} catch (error) {}
}
if (process.env.PRO_URL) {
try {
const urlData = new URL(process.env.PRO_URL);
list.push(urlData.hostname);
} catch (error) {}
}
return list;
})();
export const validateFileUrlDomain = (url: string): boolean => {
try {
// Allow all URLs if the whitelist is empty
if ((global.systemEnv?.fileUrlWhitelist || []).length === 0) {
return true;
}
const whitelistArray = [...(global.systemEnv?.fileUrlWhitelist || []), ...systemWhiteList];
const urlObj = new URL(url);
const isAllowed = whitelistArray.some((domain) => {
if (!domain || typeof domain !== 'string') return false;
return urlObj.hostname === domain;
});
if (!isAllowed) {
return false;
}
return true;
} catch (error) {
return true;
}
};

45
packages/service/core/ai/llm/utils.ts
View File

@ -14,6 +14,7 @@ import { addLog } from '../../../common/system/log';
import { getImageBase64 } from '../../../common/file/image/utils';
import { getS3ChatSource } from '../../../common/s3/sources/chat';
import { isInternalAddress } from '../../../common/system/utils';
import { getErrText } from '@fastgpt/global/common/error/utils';
export const filterGPTMessageByMaxContext = async ({
messages = [],
@ -166,26 +167,32 @@ export const loadRequestMessages = async ({
process.env.MULTIPLE_DATA_TO_BASE64 !== 'false' ||
isInternalAddress(imgUrl)
) {
const url = await (async () => {
if (item.key) {
try {
return await getS3ChatSource().createGetChatFileURL({
key: item.key,
external: false
});
} catch (error) {}
}
return imgUrl;
})();
const { completeBase64: base64 } = await getImageBase64(url);
try {
const url = await (async () => {
if (item.key) {
try {
return await getS3ChatSource().createGetChatFileURL({
key: item.key,
external: false
});
} catch (error) {}
}
return imgUrl;
})();
const { completeBase64: base64 } = await getImageBase64(url);
return {
...item,
image_url: {
...item.image_url,
url: base64
}
};
return {
...item,
image_url: {
...item.image_url,
url: base64
}
};
} catch (error) {
return Promise.reject(
`Cannot load image ${imgUrl}, because ${getErrText(error)}`
);
}
}
// 检查下这个图片是否可以被访问，如果不行的话，则过滤掉

17
packages/service/core/workflow/dispatch/index.ts
View File

@ -23,7 +23,7 @@ import type {
SystemVariablesType
} from '@fastgpt/global/core/workflow/runtime/type';
import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type.d';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { getErrText, UserError } from '@fastgpt/global/common/error/utils';
import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants';
import { filterPublicNodeResponseData } from '@fastgpt/global/core/chat/utils';
import {
@ -58,6 +58,7 @@ import type { MCPClient } from '../../app/mcp';
import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
import { i18nT } from '../../../../web/i18n/utils';
import { clone } from 'lodash';
import { validateFileUrlDomain } from '../../../common/security/fileUrlValidator';
type Props = Omit<
ChatDispatchProps,
@ -88,7 +89,21 @@ export async function dispatchWorkFlow({
}: Props & WorkflowUsageProps): Promise<DispatchFlowResponse> {
const { res, stream, runningUserInfo, runningAppInfo, lastInteractive, histories, query } = data;
// Check url valid
const invalidInput = query.some((item) => {
if (item.type === ChatItemValueTypeEnum.file && item.file?.url) {
if (!validateFileUrlDomain(item.file.url)) {
return true;
}
}
});
if (invalidInput) {
addLog.info('[Workflow run] Invalid file url');
return Promise.reject(new UserError('Invalid file url'));
}
// Check point
await checkTeamAIPoints(runningUserInfo.teamId);
const [{ timezone, externalProvider }, newUsageId] = await Promise.all([
getUserChatInfo(runningUserInfo.tmbId),
(() => {

2
projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx
View File

@ -527,7 +527,7 @@ const ChatBox = ({
file: {
type: file.type,
name: file.name,
url: file.url || '',
url: file.key ? undefined : file.url || '',
icon: file.icon || '',
key: file.key || ''
}

6
projects/app/src/pages/api/v1/chat/completions.ts
View File

@ -6,8 +6,10 @@ import { addLog } from '@fastgpt/service/common/system/log';
import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
import type { ChatCompletionCreateParams } from '@fastgpt/global/core/ai/type.d';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d';
import type {
ChatCompletionCreateParams,
ChatCompletionMessageParam
} from '@fastgpt/global/core/ai/type.d';
import {
getWorkflowEntryNodeIds,
getMaxHistoryLimitFromNodes,

6
projects/app/src/pages/api/v2/chat/completions.ts
View File

@ -6,8 +6,10 @@ import { addLog } from '@fastgpt/service/common/system/log';
import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
import type { ChatCompletionCreateParams } from '@fastgpt/global/core/ai/type.d';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d';
import type {
ChatCompletionCreateParams,
ChatCompletionMessageParam
} from '@fastgpt/global/core/ai/type.d';
import {
getWorkflowEntryNodeIds,
getMaxHistoryLimitFromNodes,