This PR introduces evaluation support designed specifically to track and benchmark applications built on the FastGPT platform. (#5476)

- Adds a lightweight evaluation framework for app-level tracking and benchmarking.
- Changes: 28 files, +1455 additions, -66 deletions.
- Branch: add-evaluations -> main.
- PR: https://github.com/chanzhi82020/FastGPT/pull/1

Applications built on FastGPT need repeatable, comparable benchmarks to measure regressions, track improvements, and validate releases. This initial implementation provides the primitives to define evaluation scenarios, run them against app endpoints or model components, and persist results for later analysis.

I updated the PR description to emphasize that the evaluation system is targeted at FastGPT-built apps and expanded the explanation of the core pieces so reviewers understand the scope and intended use. The new description outlines the feature intent, core components, and how results are captured and aggregated for benchmarking.

- Evaluation definitions
  - Define evaluation tasks that reference an app (app id, version, endpoint), test datasets or input cases, expected outputs (when applicable), and run configuration (parallelism, timeouts).
  - Support for custom metric plugins so teams can add domain-specific measures.

- Runner / Executor
  - Executes evaluation cases against app endpoints or internal model interfaces.
  - Captures raw responses, response times, status codes, and any runtime errors.
  - Computes per-case metrics (e.g., correctness, latency) immediately after each case run.

- Metrics & Aggregation
  - Built-in metrics: accuracy/success rate, latency (p50/p90/p99), throughput, error rate.
  - Aggregation produces per-run summaries and per-app historical summaries for trend analysis.
  - Allows combining metrics into composite scores for high-level benchmarking.

- Persistence & Logging
  - Stores run results, input/output pairs (when needed), timestamps, environment info, and app/version metadata so runs are reproducible and auditable.
  - Logs are retained to facilitate debugging and root-cause analysis of regressions.

- Reporting & Comparison
  - Produces aggregated reports suitable for CI gating, release notes, or dashboards.
  - Supports comparing multiple app versions or deployments side-by-side.

- Extensibility & Integration
  - Designed to plug into CI (automated runs on PRs or releases), dashboards, and downstream analysis tools.
  - Easy to add new metrics, evaluators, or dataset connectors.

By centering the evaluation system on FastGPT apps, teams can benchmark full application behavior (not only raw model outputs), correlate metrics with deployment configurations, and make informed release decisions.

- Expand built-in metric suite (e.g., F1, BLEU/ROUGE where applicable), add dataset connectors, and provide example evaluation scenarios for sample apps.
- Integrate with CI pipelines and add basic dashboarding for trend visualization.

Related Issue: N/A

Co-authored-by: Archer <545436317@qq.com>

packages/global/core/evaluation/constants.ts

@@ -1,4 +1,4 @@
-import { i18nT } from '../../../../web/i18n/utils';
+import { i18nT } from '../../../web/i18n/utils';
 
 export const evaluationFileErrors = i18nT('dashboard_evaluation:eval_file_check_error');
 

packages/global/core/evaluation/utils.ts

@@ -1,4 +1,4 @@
-import type { VariableItemType } from '../type';
+import type { VariableItemType } from '../app/type';
 
 export const getEvaluationFileHeader = (appVariables?: VariableItemType[]) => {
   if (!appVariables || appVariables.length === 0) return '*q,*a,history';

packages/service/core/app/controller.ts

@@ -7,8 +7,8 @@ import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node'
 import { encryptSecretValue, storeSecretValue } from '../../common/secret/utils';
 import { SystemToolInputTypeEnum } from '@fastgpt/global/core/app/systemTool/constants';
 import { type ClientSession } from '../../common/mongo';
-import { MongoEvaluation } from './evaluation/evalSchema';
-import { removeEvaluationJob } from './evaluation/mq';
+import { MongoEvaluation } from '../evaluation/evalSchema';
+import { removeEvaluationJob } from '../evaluation/mq';
 import { deleteChatFiles } from '../chat/controller';
 import { MongoChatItem } from '../chat/chatItemSchema';
 import { MongoChat } from '../chat/chatSchema';

packages/service/core/evaluation/evalItemSchema.ts

@@ -1,10 +1,10 @@
-import { connectionMongo, getMongoModel } from '../../../common/mongo';
+import { connectionMongo, getMongoModel } from '../../common/mongo';
 import { EvaluationCollectionName } from './evalSchema';
 import {
   EvaluationStatusEnum,
   EvaluationStatusValues
-} from '@fastgpt/global/core/app/evaluation/constants';
-import type { EvalItemSchemaType } from '@fastgpt/global/core/app/evaluation/type';
+} from '@fastgpt/global/core/evaluation/constants';
+import type { EvalItemSchemaType } from '@fastgpt/global/core/evaluation/type';
 
 const { Schema } = connectionMongo;
 

packages/service/core/evaluation/evalSchema.ts

@@ -2,10 +2,10 @@ import {
   TeamCollectionName,
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
-import { connectionMongo, getMongoModel } from '../../../common/mongo';
-import { AppCollectionName } from '../schema';
-import type { EvaluationSchemaType } from '@fastgpt/global/core/app/evaluation/type';
-import { UsageCollectionName } from '../../../support/wallet/usage/schema';
+import { connectionMongo, getMongoModel } from '../../common/mongo';
+import { AppCollectionName } from '../app/schema';
+import type { EvaluationSchemaType } from '@fastgpt/global/core/evaluation/type';
+import { UsageCollectionName } from '../../support/wallet/usage/schema';
 const { Schema } = connectionMongo;
 
 export const EvaluationCollectionName = 'eval';

packages/service/core/evaluation/index.ts

@@ -0,0 +1,370 @@
+import { addLog } from '../../common/system/log';
+import type { Job } from '../../common/bullmq';
+import { getEvaluationWorker, type EvaluationJobData, removeEvaluationJob } from './mq';
+import { MongoEvalItem } from './evalItemSchema';
+import { Types } from 'mongoose';
+import { dispatchWorkFlow } from '../workflow/dispatch';
+import { MongoEvaluation } from './evalSchema';
+import { getNanoid } from '@fastgpt/global/common/string/tools';
+import { getAppLatestVersion } from '../../core/app/version/controller';
+import {
+  getWorkflowEntryNodeIds,
+  storeEdges2RuntimeEdges,
+  storeNodes2RuntimeNodes
+} from '@fastgpt/global/core/workflow/runtime/utils';
+import type { UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
+import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants';
+import { WORKFLOW_MAX_RUN_TIMES } from '../../core/workflow/constants';
+import { getAppEvaluationScore } from './scoring';
+import { checkTeamAIPoints } from '../../support/permission/teamLimit';
+import { EvaluationStatusEnum } from '@fastgpt/global/core/evaluation/constants';
+import type {
+  EvalItemSchemaType,
+  EvaluationSchemaType
+} from '@fastgpt/global/core/evaluation/type';
+import type { Document } from 'mongoose';
+import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
+import {
+  InformLevelEnum,
+  SendInformTemplateCodeEnum
+} from '@fastgpt/global/support/user/inform/constants';
+import type { AppChatConfigType, AppSchema } from '@fastgpt/global/core/app/type';
+import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
+import type { StoreEdgeItemType } from '@fastgpt/global/core/workflow/type/edge';
+import { getErrText } from '@fastgpt/global/common/error/utils';
+import { formatModelChars2Points } from '../../support/wallet/usage/utils';
+import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
+import { concatUsage } from '../../support/wallet/usage/controller';
+import { MongoApp } from '../../core/app/schema';
+import { delay } from '@fastgpt/global/common/system/utils';
+import { removeDatasetCiteText } from '../../core/ai/utils';
+import { getUserChatInfoAndAuthTeamPoints } from '../../support/permission/auth/team';
+import { getRunningUserInfoByTmbId } from '../../support/user/team/utils';
+
+type AppContextType = {
+  appData: AppSchema;
+  timezone: string;
+  externalProvider: Record<string, any>;
+  nodes: StoreNodeItemType[];
+  edges: StoreEdgeItemType[];
+  chatConfig: AppChatConfigType;
+};
+
+export const initEvaluationWorker = () => {
+  addLog.info('Init Evaluation Worker...');
+  return getEvaluationWorker(processor);
+};
+
+const dealAiPointCheckError = async (evalId: string, error: any) => {
+  if (error === TeamErrEnum.aiPointsNotEnough) {
+    await MongoEvaluation.updateOne(
+      { _id: new Types.ObjectId(evalId) },
+      { $set: { errorMessage: error } }
+    );
+
+    const evaluation = await MongoEvaluation.findById(evalId).lean();
+    if (evaluation) {
+      sendInform2OneUser({
+        level: InformLevelEnum.important,
+        templateCode: 'LACK_OF_POINTS',
+        templateParam: {},
+        teamId: evaluation.teamId
+      });
+    }
+    return;
+  }
+
+  return Promise.reject(error);
+};
+
+const finishEvaluation = async (evalId: string) => {
+  // Computed all eval score and add to evaluation collection
+  const scoreResult = await MongoEvalItem.aggregate([
+    {
+      $match: {
+        evalId: new Types.ObjectId(evalId),
+        status: EvaluationStatusEnum.completed,
+        errorMessage: { $exists: false },
+        score: { $exists: true }
+      }
+    },
+    {
+      $group: {
+        _id: null,
+        avgScore: { $avg: '$score' }
+      }
+    }
+  ]);
+
+  const avgScore = scoreResult.length > 0 ? scoreResult[0].avgScore : 0;
+
+  await MongoEvaluation.updateOne(
+    { _id: new Types.ObjectId(evalId) },
+    {
+      $set: {
+        finishTime: new Date(),
+        score: avgScore
+      }
+    }
+  );
+
+  addLog.info('[Evaluation] Task finished', { evalId, avgScore });
+};
+
+const handleEvalItemError = async (
+  evalItem: Document<unknown, {}, EvalItemSchemaType> & EvalItemSchemaType,
+  error: any
+) => {
+  const errorMessage = getErrText(error);
+
+  await MongoEvalItem.updateOne(
+    { _id: evalItem._id },
+    {
+      $inc: { retry: -1 },
+      $set: {
+        errorMessage
+      }
+    }
+  );
+};
+
+const createMergedEvaluationUsage = async (
+  params: {
+    evaluation: EvaluationSchemaType;
+    totalPoints: number;
+  } & (
+    | {
+        type: 'run';
+      }
+    | {
+        type: 'eval';
+        inputTokens: number;
+        outputTokens: number;
+      }
+  )
+) => {
+  const { evaluation, totalPoints } = params;
+
+  if (params.type === 'run') {
+    await concatUsage({
+      billId: evaluation.usageId,
+      teamId: evaluation.teamId,
+      tmbId: evaluation.tmbId,
+      totalPoints,
+      count: 1,
+      listIndex: 0
+    });
+  } else if (params.type === 'eval') {
+    await concatUsage({
+      billId: evaluation.usageId,
+      teamId: evaluation.teamId,
+      tmbId: evaluation.tmbId,
+      totalPoints,
+      inputTokens: params.inputTokens,
+      outputTokens: params.outputTokens,
+      listIndex: 1
+    });
+  }
+};
+
+const processEvalItem = async ({
+  evalItem,
+  evaluation,
+  appContext
+}: {
+  evalItem: Document<unknown, {}, EvalItemSchemaType> & EvalItemSchemaType;
+  evaluation: EvaluationSchemaType;
+  appContext: AppContextType;
+}) => {
+  const getAppAnswer = async (): Promise<string> => {
+    if (evalItem?.response) {
+      return evalItem.response;
+    }
+
+    const { appData, timezone, externalProvider, nodes, edges, chatConfig } = appContext;
+    const chatId = getNanoid();
+
+    const query: UserChatItemValueItemType[] = [
+      {
+        type: ChatItemValueTypeEnum.text,
+        text: {
+          content: evalItem?.question || ''
+        }
+      }
+    ];
+
+    const histories = (() => {
+      try {
+        return evalItem?.history ? JSON.parse(evalItem.history) : [];
+      } catch (error) {
+        return [];
+      }
+    })();
+
+    const { assistantResponses, flowUsages } = await dispatchWorkFlow({
+      chatId,
+      timezone,
+      externalProvider,
+      mode: 'chat',
+      runningAppInfo: {
+        id: String(appData._id),
+        teamId: String(appData.teamId),
+        tmbId: String(appData.tmbId)
+      },
+      runningUserInfo: await getRunningUserInfoByTmbId(evaluation.tmbId),
+      uid: String(evaluation.tmbId),
+      runtimeNodes: storeNodes2RuntimeNodes(nodes, getWorkflowEntryNodeIds(nodes)),
+      runtimeEdges: storeEdges2RuntimeEdges(edges),
+      variables: evalItem?.globalVariables || {},
+      query,
+      chatConfig,
+      histories,
+      stream: false,
+      maxRunTimes: WORKFLOW_MAX_RUN_TIMES
+    });
+    const totalPoints = flowUsages.reduce((sum, item) => sum + (item.totalPoints || 0), 0);
+    const appAnswer = removeDatasetCiteText(assistantResponses[0]?.text?.content || '', false);
+
+    evalItem.response = appAnswer;
+    evalItem.responseTime = new Date();
+    await evalItem.save();
+
+    // Push usage
+    createMergedEvaluationUsage({
+      evaluation,
+      totalPoints,
+      type: 'run'
+    });
+
+    return appAnswer;
+  };
+
+  const appAnswer = await getAppAnswer();
+
+  // Eval score
+  const { accuracyScore, usage } = await getAppEvaluationScore({
+    question: evalItem?.question || '',
+    appAnswer,
+    standardAnswer: evalItem?.expectedResponse || '',
+    model: evaluation.evalModel
+  });
+
+  evalItem.status = EvaluationStatusEnum.completed;
+  evalItem.accuracy = accuracyScore;
+  evalItem.score = accuracyScore;
+  evalItem.finishTime = new Date();
+  await evalItem.save();
+
+  // Push usage
+  const { totalPoints: evalModelPoints } = formatModelChars2Points({
+    model: evaluation.evalModel,
+    modelType: ModelTypeEnum.llm,
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens
+  });
+  createMergedEvaluationUsage({
+    evaluation,
+    totalPoints: evalModelPoints,
+    type: 'eval',
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens
+  });
+};
+
+const processor = async (job: Job<EvaluationJobData>) => {
+  const { evalId } = job.data;
+
+  // 初始化检查
+  const evaluation = await MongoEvaluation.findById(evalId);
+  if (!evaluation) {
+    addLog.warn('[Evaluation] Eval not found', { evalId });
+    await removeEvaluationJob(evalId);
+    return;
+  }
+
+  const appData = await MongoApp.findById(evaluation.appId);
+  if (!appData) {
+    addLog.warn('[Evaluation] App not found', { evalId });
+    await removeEvaluationJob(evalId);
+    return;
+  }
+
+  const [{ timezone, externalProvider }, { nodes, edges, chatConfig }] = await Promise.all([
+    getUserChatInfoAndAuthTeamPoints(appData.tmbId),
+    getAppLatestVersion(appData._id, appData),
+    // Reset error message
+    MongoEvaluation.updateOne({ _id: new Types.ObjectId(evalId) }, { $set: { errorMessage: null } })
+  ]);
+
+  const appContext: AppContextType = {
+    appData,
+    timezone,
+    externalProvider,
+    nodes,
+    edges,
+    chatConfig
+  };
+
+  // 主循环
+  while (true) {
+    try {
+      await checkTeamAIPoints(evaluation.teamId);
+    } catch (error) {
+      return await dealAiPointCheckError(evalId, error);
+    }
+
+    const evalItem = await MongoEvalItem.findOneAndUpdate(
+      {
+        evalId,
+        status: { $in: [EvaluationStatusEnum.queuing, EvaluationStatusEnum.evaluating] },
+        retry: { $gt: 0 }
+      },
+      {
+        $set: { status: EvaluationStatusEnum.evaluating }
+      }
+    );
+    if (!evalItem) {
+      await finishEvaluation(evalId);
+      break;
+    }
+
+    // Process eval item
+    try {
+      await processEvalItem({
+        evalItem,
+        evaluation,
+        appContext
+      });
+    } catch (error) {
+      if (error === 'Evaluation model not found') {
+        addLog.warn('[Evaluation] Model not found', { evalId, model: evaluation.evalModel });
+
+        await MongoEvaluation.updateOne(
+          { _id: new Types.ObjectId(evalId) },
+          { $set: { errorMessage: `Model ${evaluation.evalModel} not found` } }
+        ).catch();
+
+        break;
+      }
+
+      await handleEvalItemError(evalItem, error);
+      await delay(100);
+    }
+  }
+};
+function getMessageTemplate(templateCode: any): {
+  getInformTemplate: any;
+  lockMinutes: any;
+  isSendQueue: any;
+} {
+  throw new Error('Function not implemented.');
+}
+
+function sendInform2OneUser(arg0: {
+  level: InformLevelEnum;
+  templateCode: string;
+  templateParam: {};
+  teamId: string;
+}) {
+  addLog.warn('sendInform2OneUser: Starting notification process:', arg0);
+}

packages/service/core/evaluation/mq.ts

@@ -1,6 +1,6 @@
-import { getQueue, getWorker, QueueNames } from '../../../common/bullmq';
+import { getQueue, getWorker, QueueNames } from '../../common/bullmq';
 import { type Processor } from 'bullmq';
-import { addLog } from '../../../common/system/log';
+import { addLog } from '../../common/system/log';
 
 export type EvaluationJobData = {
   evalId: string;

packages/service/core/evaluation/scoring.ts

@@ -0,0 +1,129 @@
+import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type';
+import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
+import { getLLMModel } from '../../core/ai/model';
+import { createChatCompletion } from '../../core/ai/config';
+import { formatLLMResponse, llmCompletionsBodyFormat } from '../../core/ai/utils';
+import { loadRequestMessages } from '../../core/chat/utils';
+import { countGptMessagesTokens, countPromptTokens } from '../../common/string/tiktoken';
+
+const template_accuracy1 = `
+Instruction: You are a world class state of the art assistant for rating a User Answer given a Question. The Question is completely answered by the Reference Answer.
+Say 4, if User Answer is full contained and equivalent to Reference Answer in all terms, topics, numbers, metrics, dates and units.
+Say 2, if User Answer is partially contained and almost equivalent to Reference Answer in all terms, topics, numbers, metrics, dates and units.
+Say 0, if User Answer is not contained in Reference Answer or not accurate in all terms, topics, numbers, metrics, dates and units or the User Answer do not answer the question.
+Do not explain or justify your rating. Your rating must be only 4, 2 or 0 according to the instructions above.
+
+## Question
+{query}
+
+## Answer0
+{sentence_inference}
+
+## Answer1
+{sentence_true}
+
+## Rating`;
+
+const template_accuracy2 = `
+I will rate the User Answer in comparison to the Reference Answer for a given Question.
+A rating of 4 indicates that the User Answer is entirely consistent with the Reference Answer, covering all aspects, topics, numbers, metrics, dates, and units.
+A rating of 2 signifies that the User Answer is mostly aligned with the Reference Answer, with minor discrepancies in some areas.
+A rating of 0 means that the User Answer is either inaccurate, incomplete, or unrelated to the Reference Answer, or it fails to address the Question.
+I will provide the rating without any explanation or justification, adhering to the following scale: 0 (no match), 2 (partial match), 4 (exact match).
+Do not explain or justify my rating. My rating must be only 4, 2 or 0 only.
+
+## Question
+{query}
+
+## Answer0
+{sentence_inference}
+
+## Answer1
+{sentence_true}
+
+## Rating`;
+
+export const getAppEvaluationScore = async ({
+  question,
+  appAnswer,
+  standardAnswer,
+  model
+}: {
+  question: string;
+  appAnswer: string;
+  standardAnswer: string;
+  model: string;
+}) => {
+  const modelData = getLLMModel(model);
+  if (!modelData) {
+    return Promise.reject('Evaluation model not found');
+  }
+
+  const getEvalResult = async (template: string) => {
+    const messages: ChatCompletionMessageParam[] = [
+      {
+        role: ChatCompletionRequestMessageRoleEnum.System,
+        content: template
+      },
+      {
+        role: ChatCompletionRequestMessageRoleEnum.User,
+        content: [
+          {
+            type: 'text',
+            text: `## Question
+${question}
+
+## Answer0
+${appAnswer}
+
+## Answer1
+${standardAnswer}
+
+## Rating`
+          }
+        ]
+      }
+    ];
+    const { response } = await createChatCompletion({
+      body: llmCompletionsBodyFormat(
+        {
+          model: modelData.model,
+          temperature: 0.3,
+          messages: await loadRequestMessages({ messages, useVision: true }),
+          stream: true,
+          max_tokens: 5
+        },
+        modelData
+      )
+    });
+
+    const { text, usage } = await formatLLMResponse(response);
+
+    const numberText = Number(text);
+    const rate = isNaN(numberText) ? 0 : numberText / 4;
+
+    return {
+      rate,
+      inputTokens: usage?.prompt_tokens || (await countGptMessagesTokens(messages)),
+      outputTokens: usage?.completion_tokens || (await countPromptTokens(text))
+    };
+  };
+
+  const results = await Promise.all([
+    getEvalResult(template_accuracy1),
+    getEvalResult(template_accuracy2)
+  ]);
+
+  const accuracyScore =
+    Math.round((results.reduce((acc, item) => acc + item.rate, 0) / results.length) * 100) / 100;
+  const inputTokens = results.reduce((acc, item) => acc + item.inputTokens, 0);
+  const outputTokens = results.reduce((acc, item) => acc + item.outputTokens, 0);
+
+  return {
+    accuracyScore,
+    usage: {
+      inputTokens,
+      outputTokens
+    }
+  };
+};

packages/service/core/evaluation/utils.ts

@@ -1,8 +1,15 @@
-import { evaluationFileErrors } from '@fastgpt/global/core/app/evaluation/constants';
-import { getEvaluationFileHeader } from '@fastgpt/global/core/app/evaluation/utils';
-import type { VariableItemType } from '@fastgpt/global/core/app/type';
-import { addLog } from '../../../common/system/log';
 import { VariableInputEnum } from '@fastgpt/global/core/workflow/constants';
+import { evaluationFileErrors } from '@fastgpt/global/core/evaluation/constants';
+import { getEvaluationFileHeader } from '@fastgpt/global/core/evaluation/utils';
+import type { VariableItemType } from '@fastgpt/global/core/app/type';
+// import { addLog } from '@fastgpt/service/common/system/log';
+import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
+import { Types } from 'mongoose';
+import { retryFn } from '@fastgpt/global/common/system/utils';
+import { i18nT } from '../../../web/i18n/utils';
+import { addLog } from '../../common/system/log';
+import { MongoEvaluation } from './evalSchema';
+import { addEvaluationJob } from './mq';
 import Papa from 'papaparse';
 
 export const parseEvaluationCSV = (rawText: string) => {
@@ -24,15 +31,27 @@ export const validateEvaluationFile = async (
   rawText: string,
   appVariables?: VariableItemType[]
 ) => {
-  // Parse CSV using Papa Parse
-  const csvData = parseEvaluationCSV(rawText);
-  const dataLength = csvData.length;
+  // const lines = rawText.trim().split('\r\n');
+  // const dataLength = lines.length;
+
+  // 使用正则表达式分割所有类型的换行符（\r\n、\n、\r）
+  const lines = rawText.trim().split(/\r?\n|\r/);
+  const dataLength = lines.length;
+
+  // 过滤可能的空行（处理文件末尾可能的空行）
+  const nonEmptyLines = lines.filter((line) => line.trim() !== '');
+  if (nonEmptyLines.length === 0) {
+    addLog.error('File is empty');
+    return Promise.reject(evaluationFileErrors);
+  }
 
   // Validate file header
   const expectedHeader = getEvaluationFileHeader(appVariables);
-  const actualHeader = csvData[0]?.join(',') || '';
+  // 去除头部可能的空白字符（如BOM头或空格）
+  const actualHeader = nonEmptyLines[0].trim();
+
   if (actualHeader !== expectedHeader) {
-    addLog.error(`Header mismatch. Expected: ${expectedHeader}, Got: ${actualHeader}`);
+    addLog.error(`Header mismatch. Expected: "${expectedHeader}", Got: "${actualHeader}"`);
     return Promise.reject(evaluationFileErrors);
   }
 
@@ -48,7 +67,7 @@ export const validateEvaluationFile = async (
     return Promise.reject(evaluationFileErrors);
   }
 
-  const headers = csvData[0];
+  const headers = lines[0].split(',');
 
   // Get required field indices
   const requiredFields = headers
@@ -58,8 +77,8 @@ export const validateEvaluationFile = async (
   const errors: string[] = [];
 
   // Validate each data row
-  for (let i = 1; i < csvData.length; i++) {
-    const values = csvData[i];
+  for (let i = 1; i < lines.length; i++) {
+    const values = lines[i].trim().split(',');
 
     // Check required fields
     requiredFields.forEach(({ header, index }) => {
@@ -84,7 +103,7 @@ export const validateEvaluationFile = async (
     return Promise.reject(evaluationFileErrors);
   }
 
-  return { csvData, dataLength };
+  return { lines, dataLength };
 };
 
 const validateRowVariables = ({
@@ -145,3 +164,38 @@ const validateRowVariables = ({
     }
   });
 };
+
+export const checkTeamHasRunningEvaluation = async (teamId: string) => {
+  const runningEvaluation = await MongoEvaluation.findOne(
+    {
+      teamId: new Types.ObjectId(teamId),
+      finishTime: { $exists: false }
+    },
+    '_id'
+  ).lean();
+
+  if (runningEvaluation) {
+    return Promise.reject(i18nT('dashboard_evaluation:team_has_running_evaluation'));
+  }
+};
+
+export const resumePausedEvaluations = async (teamId: string): Promise<any> => {
+  return retryFn(async () => {
+    const pausedEvaluations = await MongoEvaluation.find({
+      teamId: new Types.ObjectId(teamId),
+      errorMessage: TeamErrEnum.aiPointsNotEnough,
+      finishTime: { $exists: false }
+    }).lean();
+
+    if (pausedEvaluations.length === 0) {
+      return;
+    }
+
+    for (const evaluation of pausedEvaluations) {
+      await MongoEvaluation.updateOne({ _id: evaluation._id }, { $unset: { errorMessage: 1 } });
+      await addEvaluationJob({ evalId: String(evaluation._id) });
+    }
+
+    addLog.info('Resumed paused evaluations', { teamId, count: pausedEvaluations.length });
+  }, 3);
+};

packages/service/support/permission/evaluation/auth.ts

@@ -3,10 +3,10 @@ import {
   ManagePermissionVal,
   ReadPermissionVal
 } from '@fastgpt/global/support/permission/constant';
-import type { EvaluationSchemaType } from '@fastgpt/global/core/app/evaluation/type';
+import type { EvaluationSchemaType } from '@fastgpt/global/core/evaluation/type';
 import type { AuthModeType } from '../type';
-import { MongoEvaluation } from '../../../core/app/evaluation/evalSchema';
 import { parseHeaderCert } from '../auth/common';
+import { MongoEvaluation } from '../../../core/evaluation/evalSchema';
 
 export const authEval = async ({
   evalId,

packages/service/type/env.d.ts

@@ -46,6 +46,10 @@ declare global {
       CHAT_LOG_SOURCE_ID_PREFIX?: string;
 
       NEXT_PUBLIC_BASE_URL: string;
+
+      // evaluations settings
+      EVAL_CONCURRENCY?: string;
+      EVAL_LINE_LIMIT?: string;
     }
   }
 }

projects/app/.env.template

@@ -101,3 +101,7 @@ CONFIG_JSON_PATH=
 # CHAT_LOG_SOURCE_ID_PREFIX=fastgpt-
 
 
+# evaluations settings
+EVAL_CONCURRENCY=3 # the number of concurrent evaluations tasks
+EVAL_LINE_LIMIT=1000 # the max line number of the uploaded eval data file
+

projects/app/src/instrumentation.ts

@@ -15,6 +15,7 @@ export async function register() {
         { initVectorStore },
         { initRootUser },
         { startMongoWatch },
+        { initEvaluationWorker },
         { startCron },
         { startTrainingQueue },
         { preLoadWorker },
@@ -29,6 +30,7 @@ export async function register() {
         import('@fastgpt/service/common/vectorDB/controller'),
         import('@/service/mongo'),
         import('@/service/common/system/volumnMongoWatch'),
+        import('@fastgpt/service/core/evaluation'),
         import('@/service/common/system/cron'),
         import('@/service/core/dataset/training/utils'),
         import('@fastgpt/service/worker/preload'),
@@ -59,6 +61,7 @@ export async function register() {
       ]);
 
       startMongoWatch();
+      initEvaluationWorker();
       startCron();
       startTrainingQueue(true);
 

projects/app/src/pageComponents/account/model/AddModelBox.tsx

@@ -677,16 +677,14 @@ export const ModelEditModal = ({
                       </Flex>
                     </Td>
                   </Tr>
-                  {feConfigs?.isPlus && (
-                    <Tr>
-                      <Td>{t('account_model:use_in_eval')}</Td>
-                      <Td textAlign={'right'}>
-                        <Flex justifyContent={'flex-end'}>
-                          <Switch {...register('useInEvaluation')} />
-                        </Flex>
-                      </Td>
-                    </Tr>
-                  )}
+                  <Tr>
+                    <Td>{t('account_model:use_in_eval')}</Td>
+                    <Td textAlign={'right'}>
+                      <Flex justifyContent={'flex-end'}>
+                        <Switch {...register('useInEvaluation')} />
+                      </Flex>
+                    </Td>
+                  </Tr>
                   <Tr>
                     <Td>
                       <HStack spacing={1}>

projects/app/src/pageComponents/dashboard/Container.tsx

@@ -191,21 +191,16 @@ const DashboardContainer = ({
         groupName: t('common:mcp_server'),
         children: []
       },
-      ...(feConfigs?.isPlus
-        ? [
-            {
-              groupId: TabEnum.evaluation,
-              groupAvatar: 'kbTest',
-              groupName: t('common:app_evaluation'),
-              children: []
-            }
-          ]
-        : [])
+      {
+        groupId: TabEnum.evaluation,
+        groupAvatar: 'kbTest',
+        groupName: t('common:app_evaluation'),
+        children: []
+      }
     ];
   }, [
     currentType,
     feConfigs.appTemplateCourse,
-    feConfigs?.isPlus,
     i18n.language,
     pluginGroups,
     t,

projects/app/src/pageComponents/evaluation/DetailModal.tsx

@@ -23,7 +23,7 @@ import {
   getEvalItemsList,
   retryEvalItem,
   updateEvalItem
-} from '@/web/core/app/api/evaluation';
+} from '@/web/core/evaluation/evaluation';
 import { usePagination } from '@fastgpt/web/hooks/usePagination';
 import { downloadFetch, getWebLLMModel } from '@/web/common/system/utils';
 import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConfirm';
@@ -33,9 +33,9 @@ import { useForm } from 'react-hook-form';
 import {
   EvaluationStatusMap,
   EvaluationStatusEnum
-} from '@fastgpt/global/core/app/evaluation/constants';
-import type { evaluationType, listEvalItemsItem } from '@fastgpt/global/core/app/evaluation/type';
-import type { updateEvalItemBody } from '@fastgpt/global/core/app/evaluation/api';
+} from '@fastgpt/global/core/evaluation/constants';
+import type { evaluationType, listEvalItemsItem } from '@fastgpt/global/core/evaluation/type';
+import type { updateEvalItemBody } from '@fastgpt/global/core/evaluation/api';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 
 const formatEvaluationStatus = (item: { status: number; errorMessage?: string }, t: TFunction) => {
@@ -129,7 +129,7 @@ const EvaluationDetailModal = ({
 
   const { runAsync: exportEval, loading: isDownloading } = useRequest2(async () => {
     await downloadFetch({
-      url: `/api/proApi/core/app/evaluation/exportItems?evalId=${evalDetail._id}`,
+      url: `/api/core/evaluation/exportItems?evalId=${evalDetail._id}`,
       filename: `${evalDetail.name}.csv`,
       body: {
         title: t('dashboard_evaluation:evaluation_export_title'),

projects/app/src/pages/api/core/evaluation/create.ts

@@ -0,0 +1,159 @@
+import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middleware/entry';
+import { addLog } from '@fastgpt/service/common/system/log';
+import { removeFilesByPaths } from '@fastgpt/service/common/file/utils';
+import { getUploadModel } from '@fastgpt/service/common/file/multer';
+import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { authApp } from '@fastgpt/service/support/permission/app/auth';
+import { readRawTextByLocalFile } from '@fastgpt/service/common/file/read/utils';
+import { createEvaluationUsage } from '@fastgpt/service/support/wallet/usage/controller';
+import { MongoEvaluation } from '@fastgpt/service/core/evaluation/evalSchema';
+import { MongoEvalItem } from '@fastgpt/service/core/evaluation/evalItemSchema';
+import { addEvaluationJob } from '@fastgpt/service/core/evaluation/mq';
+import { addAuditLog, getI18nAppType } from '@fastgpt/service/support/user/audit/util';
+import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
+import { validateEvaluationFile } from '@fastgpt/service/core/evaluation/utils';
+import { EvaluationStatusEnum } from '@fastgpt/global/core/evaluation/constants';
+import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
+import { checkTeamAIPoints } from '@fastgpt/service/support/permission/teamLimit';
+import { checkTeamHasRunningEvaluation } from '@fastgpt/service/core/evaluation/utils';
+
+export type createEvaluationBody = {
+  name: string;
+  appId: string;
+  evalModel: string;
+};
+
+const MAX_EVAL_ITEMS = process.env.EVAL_LINE_LIMIT ? Number(process.env.EVAL_LINE_LIMIT) : 1000;
+
+async function handler(req: ApiRequestProps<createEvaluationBody, {}>, res: ApiResponseType<any>) {
+  const filePaths: string[] = [];
+
+  try {
+    const upload = getUploadModel({
+      maxSize: global.feConfigs?.uploadFileMaxSize
+    });
+
+    const { file, data } = await upload.getUploadFile<createEvaluationBody>(req, res);
+    filePaths.push(file.path);
+
+    if (file.mimetype !== 'text/csv') {
+      return Promise.reject('File must be a CSV file');
+    }
+
+    const { teamId, tmbId, app } = await authApp({
+      req,
+      authToken: true,
+      authApiKey: true,
+      per: ReadPermissionVal,
+      appId: data.appId
+    });
+    await checkTeamAIPoints(teamId);
+    await checkTeamHasRunningEvaluation(teamId);
+
+    const { rawText } = await readRawTextByLocalFile({
+      teamId,
+      tmbId,
+      path: file.path,
+      encoding: file.encoding,
+      getFormatText: false
+    });
+    removeFilesByPaths(filePaths);
+
+    const appVariables = app.chatConfig.variables;
+
+    const { lines } = await validateEvaluationFile(rawText, appVariables);
+
+    if (lines.length - 1 > MAX_EVAL_ITEMS) {
+      return Promise.reject(`File must be less than ${MAX_EVAL_ITEMS} lines`);
+    }
+
+    const headers = lines[0].split(',');
+    const qIndex = headers.findIndex((h) => h.trim() === '*q');
+    const aIndex = headers.findIndex((h) => h.trim() === '*a');
+    const historyIndex = headers.findIndex((h) => h.trim() === 'history');
+
+    const { usageId } = await createEvaluationUsage({
+      teamId,
+      tmbId,
+      appName: app.name,
+      model: data.evalModel
+    });
+
+    const evalItems = lines.slice(1).map((line) => {
+      const values = line.split(',');
+      const question = values[qIndex];
+      const expectedResponse = values[aIndex];
+      const history = historyIndex !== -1 ? values[historyIndex] : '';
+
+      const globalVariables = headers.slice(0, qIndex).reduce(
+        (acc, header, j) => {
+          const headerName = header.trim().replace(/^\*/, '');
+          acc[headerName] = values[j] || '';
+          return acc;
+        },
+        {} as Record<string, string>
+      );
+
+      return {
+        question,
+        expectedResponse,
+        history,
+        globalVariables
+      };
+    });
+
+    await mongoSessionRun(async (session) => {
+      const [evaluation] = await MongoEvaluation.create(
+        [
+          {
+            teamId,
+            tmbId,
+            appId: data.appId,
+            usageId,
+            evalModel: data.evalModel,
+            name: data.name
+          }
+        ],
+        { session, ordered: true }
+      );
+
+      const evalItemsWithId = evalItems.map((item) => ({
+        question: item.question,
+        expectedResponse: item.expectedResponse,
+        history: item.history,
+        globalVariables: item.globalVariables,
+        evalId: evaluation._id,
+        status: EvaluationStatusEnum.queuing
+      }));
+      await MongoEvalItem.insertMany(evalItemsWithId, {
+        session,
+        ordered: false
+      });
+
+      await addEvaluationJob({ evalId: evaluation._id });
+    });
+
+    addAuditLog({
+      tmbId,
+      teamId,
+      event: AuditEventEnum.CREATE_EVALUATION,
+      params: {
+        name: data.name,
+        appName: app.name
+      }
+    });
+  } catch (error) {
+    addLog.error(`create evaluation error: ${error}`);
+    removeFilesByPaths(filePaths);
+    return Promise.reject(error);
+  }
+}
+
+export default NextAPI(handler);
+
+export const config = {
+  api: {
+    bodyParser: false
+  }
+};

projects/app/src/pages/api/core/evaluation/delete.ts

@@ -0,0 +1,53 @@
+import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middleware/entry';
+import { MongoEvaluation } from '@fastgpt/service/core/evaluation/evalSchema';
+import { MongoEvalItem } from '@fastgpt/service/core/evaluation/evalItemSchema';
+import { authEval } from '@fastgpt/service/support/permission/evaluation/auth';
+import { addAuditLog } from '@fastgpt/service/support/user/audit/util';
+import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
+import { removeEvaluationJob } from '@fastgpt/service/core/evaluation/mq';
+import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
+import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
+
+async function handler(req: ApiRequestProps<{}, { evalId: string }>, res: ApiResponseType<any>) {
+  const { evalId } = req.query;
+
+  const { tmbId, teamId, evaluation } = await authEval({
+    req,
+    per: WritePermissionVal,
+    evalId,
+    authToken: true,
+    authApiKey: true
+  });
+
+  await mongoSessionRun(async (session) => {
+    await MongoEvaluation.deleteOne(
+      {
+        _id: evalId
+      },
+      { session }
+    );
+
+    await MongoEvalItem.deleteMany(
+      {
+        evalId
+      },
+      { session }
+    );
+
+    await removeEvaluationJob(evalId);
+  });
+
+  addAuditLog({
+    tmbId,
+    teamId,
+    event: AuditEventEnum.DELETE_EVALUATION,
+    params: {
+      name: evaluation.name
+    }
+  });
+
+  return {};
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/evaluation/deleteItem.ts

@@ -0,0 +1,23 @@
+import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middleware/entry';
+import { MongoEvalItem } from '@fastgpt/service/core/evaluation/evalItemSchema';
+import { authEval } from '@fastgpt/service/support/permission/evaluation/auth';
+import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
+
+async function handler(
+  req: ApiRequestProps<{}, { evalId: string; itemId: string }>,
+  res: ApiResponseType<any>
+) {
+  const { evalId, itemId } = req.query;
+  await authEval({
+    req,
+    per: WritePermissionVal,
+    evalId,
+    authToken: true,
+    authApiKey: true
+  });
+
+  await MongoEvalItem.deleteOne({ _id: itemId, evalId });
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/evaluation/exportItems.ts

@@ -0,0 +1,112 @@
+import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middleware/entry';
+import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { MongoEvaluation } from '@fastgpt/service/core/evaluation/evalSchema';
+import { MongoEvalItem } from '@fastgpt/service/core/evaluation/evalItemSchema';
+import { Types } from 'mongoose';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
+import { authEval } from '@fastgpt/service/support/permission/evaluation/auth';
+import { addAuditLog, getI18nAppType } from '@fastgpt/service/support/user/audit/util';
+import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
+import { generateCsv } from '@fastgpt/service/common/file/csv';
+
+export type exportItemsQuery = {
+  evalId: string;
+};
+
+export type exportItemsBody = {
+  title: string;
+  statusMap: Record<string, { label: string }>;
+};
+
+async function handler(
+  req: ApiRequestProps<exportItemsBody, exportItemsQuery>,
+  res: ApiResponseType<any>
+) {
+  const { evalId } = req.query;
+  const { title, statusMap } = req.body || {};
+
+  const { teamId, tmbId } = await authEval({
+    req,
+    per: ReadPermissionVal,
+    evalId,
+    authToken: true,
+    authApiKey: true
+  });
+
+  const evaluation = await MongoEvaluation.findById(evalId);
+  if (!evaluation) {
+    return Promise.reject('Evaluation task does not exist');
+  }
+
+  res.setHeader('Content-Type', 'text/csv; charset=utf-8;');
+  res.setHeader(
+    'Content-Disposition',
+    `attachment; filename=${encodeURIComponent(evaluation?.name || 'evaluation')}.csv;`
+  );
+
+  const evalItems = await MongoEvalItem.find(
+    {
+      evalId: new Types.ObjectId(evalId)
+    },
+    'globalVariables question expectedResponse response status accuracy relevance semanticAccuracy score errorMessage',
+    {
+      ...readFromSecondary
+    }
+  );
+
+  const allVariableKeys = new Set<string>();
+  evalItems.forEach((doc) => {
+    if (doc.globalVariables) {
+      Object.keys(doc.globalVariables).forEach((key) => allVariableKeys.add(key));
+    }
+  });
+  const variableKeysArray = Array.from(allVariableKeys).sort();
+
+  const baseHeaders = title.split(',');
+  const headers = [...variableKeysArray, ...baseHeaders];
+
+  const data = evalItems.map((doc) => {
+    const question = doc.question || '';
+    const expectedResponse = doc.expectedResponse || '';
+    const response = doc.response || '';
+
+    const status = (() => {
+      if (doc.errorMessage) {
+        return 'Error'; // Show error when errorMessage exists
+      }
+      return statusMap[doc.status]?.label || 'Unknown';
+    })();
+
+    const score = !!doc.score ? doc.score.toFixed(2) : '0';
+
+    const variableValues = variableKeysArray.map((key) => {
+      return doc.globalVariables?.[key] || '';
+    });
+
+    return [...variableValues, question, expectedResponse, response, status, score];
+  });
+
+  const csvContent = generateCsv(headers, data);
+
+  res.write('\uFEFF' + csvContent);
+
+  addAuditLog({
+    tmbId,
+    teamId,
+    event: AuditEventEnum.EXPORT_EVALUATION,
+    params: {
+      name: evaluation.name
+    }
+  });
+
+  res.end();
+}
+
+export default NextAPI(handler);
+
+export const config = {
+  api: {
+    responseLimit: '100mb'
+  }
+};

projects/app/src/pages/api/core/evaluation/list.ts

@@ -0,0 +1,194 @@
+import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middleware/entry';
+import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
+import {
+  ReadPermissionVal,
+  PerResourceTypeEnum
+} from '@fastgpt/global/support/permission/constant';
+import { MongoEvaluation } from '@fastgpt/service/core/evaluation/evalSchema';
+import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
+import { Types } from '@fastgpt/service/common/mongo';
+import type { PaginationResponse } from '@fastgpt/web/common/fetch/type';
+import type { listEvaluationsBody } from '@fastgpt/global/core/evaluation/api';
+import type { EvaluationSchemaType, evaluationType } from '@fastgpt/global/core/evaluation/type';
+import { replaceRegChars } from '@fastgpt/global/common/string/tools';
+import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
+import { getGroupsByTmbId } from '@fastgpt/service/support/permission/memberGroup/controllers';
+import { getOrgIdSetWithParentByTmbId } from '@fastgpt/service/support/permission/org/controllers';
+import type { TeamMemberSchema } from '@fastgpt/global/support/user/team/type';
+import type { AppSchema } from '@fastgpt/global/core/app/type';
+import { i18nT } from '@fastgpt/web/i18n/utils';
+import { MongoApp } from '@fastgpt/service/core/app/schema';
+
+async function handler(
+  req: ApiRequestProps<listEvaluationsBody, {}>,
+  res: ApiResponseType<any>
+): Promise<PaginationResponse<evaluationType>> {
+  const {
+    teamId,
+    tmbId,
+    permission: teamPer
+  } = await authUserPer({
+    req,
+    authToken: true,
+    authApiKey: true,
+    per: ReadPermissionVal
+  });
+
+  const { offset, pageSize } = parsePaginationRequest(req);
+  const { searchKey } = req.body;
+
+  const [perList, myGroupMap, myOrgSet] = await Promise.all([
+    MongoResourcePermission.find({
+      resourceType: PerResourceTypeEnum.app,
+      teamId,
+      resourceId: {
+        $exists: true
+      }
+    }).lean(),
+    getGroupsByTmbId({
+      tmbId,
+      teamId
+    }).then((item) => {
+      const map = new Map<string, 1>();
+      item.forEach((item) => {
+        map.set(String(item._id), 1);
+      });
+      return map;
+    }),
+    getOrgIdSetWithParentByTmbId({
+      teamId,
+      tmbId
+    })
+  ]);
+  const myPerAppIdList = perList
+    .filter(
+      (item) =>
+        String(item.tmbId) === String(tmbId) ||
+        myGroupMap.has(String(item.groupId)) ||
+        myOrgSet.has(String(item.orgId))
+    )
+    .map((item) => new Types.ObjectId(item.resourceId));
+
+  const myAppIds = await MongoApp.find({
+    teamId: new Types.ObjectId(teamId),
+    $or: [{ tmbId }, { parentId: { $in: myPerAppIdList } }]
+  })
+    .select('_id')
+    .lean();
+
+  const match = {
+    teamId: new Types.ObjectId(teamId),
+    ...(searchKey && { name: { $regex: new RegExp(`${replaceRegChars(searchKey)}`, 'i') } }),
+    ...(!teamPer.isOwner && {
+      appId: {
+        $in: [...myPerAppIdList, ...myAppIds.map((item) => item._id)]
+      }
+    })
+  };
+
+  const [evaluations, total] = await Promise.all([
+    MongoEvaluation.aggregate(
+      buildPipeline(match, offset, pageSize)
+    ) as unknown as (EvaluationSchemaType & {
+      teamMember: TeamMemberSchema;
+      app: AppSchema;
+      stats: {
+        totalCount: number;
+        completedCount: number;
+        errorCount: number;
+        avgScore: number;
+      };
+    })[],
+    MongoEvaluation.countDocuments(match)
+  ]);
+
+  return {
+    total,
+    list: evaluations.map((item) => {
+      const { stats } = item;
+      const { totalCount = 0, completedCount = 0, errorCount = 0, avgScore } = stats || {};
+
+      const calculatedScore = totalCount === completedCount ? avgScore || 0 : undefined;
+
+      return {
+        name: item.name,
+        appId: String(item.appId),
+        createTime: item.createTime,
+        finishTime: item.finishTime,
+        evalModel: item.evalModel,
+        errorMessage: item.errorMessage,
+        score: calculatedScore,
+        _id: String(item._id),
+        executorAvatar: item.teamMember?.avatar,
+        executorName: item.teamMember?.name,
+        appAvatar: item.app?.avatar,
+        appName: item.app?.name || i18nT('app:deleted'),
+        completedCount,
+        errorCount,
+        totalCount
+      };
+    })
+  };
+}
+
+const buildPipeline = (match: Record<string, any>, offset: number, pageSize: number) => [
+  { $match: match },
+  { $sort: { createTime: -1 as const } },
+  { $skip: offset },
+  { $limit: pageSize },
+  {
+    $lookup: {
+      from: 'team_members',
+      localField: 'tmbId',
+      foreignField: '_id',
+      as: 'teamMember'
+    }
+  },
+  {
+    $lookup: {
+      from: 'apps',
+      localField: 'appId',
+      foreignField: '_id',
+      as: 'app'
+    }
+  },
+  {
+    $lookup: {
+      from: 'eval_items',
+      let: { evalId: '$_id' },
+      pipeline: [
+        { $match: { $expr: { $eq: ['$evalId', '$$evalId'] } } },
+        {
+          $group: {
+            _id: null,
+            totalCount: { $sum: 1 },
+            completedCount: {
+              $sum: { $cond: [{ $eq: ['$status', 2] }, 1, 0] }
+            },
+            errorCount: {
+              $sum: {
+                $cond: [{ $ifNull: ['$errorMessage', false] }, 1, 0]
+              }
+            },
+            avgScore: {
+              $avg: {
+                $cond: [{ $ne: ['$score', null] }, '$score', '$$REMOVE']
+              }
+            }
+          }
+        }
+      ],
+      as: 'evalStats'
+    }
+  },
+  {
+    $addFields: {
+      teamMember: { $arrayElemAt: ['$teamMember', 0] },
+      app: { $arrayElemAt: ['$app', 0] },
+      stats: { $arrayElemAt: ['$evalStats', 0] }
+    }
+  }
+];
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/evaluation/listItems.ts

@@ -0,0 +1,83 @@
+import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middleware/entry';
+import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
+import { MongoEvalItem } from '@fastgpt/service/core/evaluation/evalItemSchema';
+import { Types } from 'mongoose';
+import { authEval } from '@fastgpt/service/support/permission/evaluation/auth';
+import type { listEvalItemsBody } from '@fastgpt/global/core/evaluation/api';
+import type { listEvalItemsItem } from '@fastgpt/global/core/evaluation/type';
+import type { PaginationResponse } from '@fastgpt/web/common/fetch/type';
+import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+
+async function handler(
+  req: ApiRequestProps<listEvalItemsBody, {}>,
+  res: ApiResponseType<any>
+): Promise<PaginationResponse<listEvalItemsItem>> {
+  const { evalId } = req.body;
+  await authEval({
+    req,
+    per: ReadPermissionVal,
+    evalId,
+    authToken: true,
+    authApiKey: true
+  });
+  const { offset, pageSize } = parsePaginationRequest(req);
+
+  const aggregationPipeline = [
+    {
+      $match: {
+        evalId: new Types.ObjectId(evalId)
+      }
+    },
+    {
+      $addFields: {
+        sortStatus: {
+          $switch: {
+            branches: [
+              { case: { $ifNull: ['$errorMessage', false] }, then: 0 },
+              { case: { $eq: ['$status', 1] }, then: 1 },
+              { case: { $eq: ['$status', 0] }, then: 2 },
+              { case: { $eq: ['$status', 2] }, then: 3 }
+            ],
+            default: 4
+          }
+        }
+      }
+    },
+    {
+      $sort: { sortStatus: 1 as const, _id: 1 as const }
+    },
+    {
+      $skip: offset
+    },
+    {
+      $limit: pageSize
+    }
+  ];
+
+  const [result, total] = await Promise.all([
+    MongoEvalItem.aggregate(aggregationPipeline),
+    MongoEvalItem.countDocuments({ evalId })
+  ]);
+
+  return {
+    total,
+    list: result.map((item) => ({
+      evalItemId: String(item._id),
+      evalId: String(item.evalId),
+      retry: item.retry,
+      question: item.question,
+      expectedResponse: item.expectedResponse,
+      response: item.response,
+      globalVariables: item.globalVariables,
+      status: item.status,
+      errorMessage: item.errorMessage,
+      accuracy: item.accuracy,
+      relevance: item.relevance,
+      semanticAccuracy: item.semanticAccuracy,
+      score: item.score
+    }))
+  };
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/evaluation/retryItem.ts

@@ -0,0 +1,45 @@
+import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middleware/entry';
+import { authEval } from '@fastgpt/service/support/permission/evaluation/auth';
+import { MongoEvalItem } from '@fastgpt/service/core/evaluation/evalItemSchema';
+import { checkEvaluationJobActive, addEvaluationJob } from '@fastgpt/service/core/evaluation/mq';
+import { EvaluationStatusEnum } from '@fastgpt/global/core/evaluation/constants';
+import type { retryEvalItemBody } from '@fastgpt/global/core/evaluation/api';
+import { checkTeamAIPoints } from '@fastgpt/service/support/permission/teamLimit';
+import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
+
+async function handler(req: ApiRequestProps<retryEvalItemBody, {}>, res: ApiResponseType<any>) {
+  const { evalItemId } = req.body;
+
+  const evaluationItem = await MongoEvalItem.findById(evalItemId);
+  if (!evaluationItem) return Promise.reject('evaluationItem not found');
+
+  const { teamId, evaluation } = await authEval({
+    req,
+    per: WritePermissionVal,
+    evalId: evaluationItem.evalId,
+    authToken: true,
+    authApiKey: true
+  });
+
+  await checkTeamAIPoints(teamId);
+
+  await MongoEvalItem.updateOne(
+    { _id: evalItemId },
+    {
+      $set: {
+        status: EvaluationStatusEnum.queuing,
+        errorMessage: null,
+        response: null,
+        accuracy: null,
+        relevance: null,
+        semanticAccuracy: null,
+        score: null,
+        retry: 3
+      }
+    }
+  );
+  await addEvaluationJob({ evalId: evaluation._id });
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/evaluation/updateItem.ts

@@ -0,0 +1,46 @@
+import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middleware/entry';
+import { authEval } from '@fastgpt/service/support/permission/evaluation/auth';
+import { MongoEvalItem } from '@fastgpt/service/core/evaluation/evalItemSchema';
+import { addEvaluationJob } from '@fastgpt/service/core/evaluation/mq';
+import { EvaluationStatusEnum } from '@fastgpt/global/core/evaluation/constants';
+import type { updateEvalItemBody } from '@fastgpt/global/core/evaluation/api';
+import { checkTeamAIPoints } from '@fastgpt/service/support/permission/teamLimit';
+
+async function handler(req: ApiRequestProps<updateEvalItemBody, {}>, res: ApiResponseType<any>) {
+  const { evalItemId, question, expectedResponse, variables } = req.body;
+
+  const evaluationItem = await MongoEvalItem.findById(evalItemId);
+  if (!evaluationItem) return Promise.reject('evaluationItem not found');
+
+  const { teamId, evaluation } = await authEval({
+    req,
+    evalId: evaluationItem.evalId,
+    authToken: true,
+    authApiKey: true
+  });
+  await checkTeamAIPoints(teamId);
+
+  await MongoEvalItem.updateOne(
+    { _id: evalItemId },
+    {
+      $set: {
+        question,
+        expectedResponse,
+        status: EvaluationStatusEnum.queuing,
+        errorMessage: null,
+        response: null,
+        accuracy: null,
+        relevance: null,
+        semanticAccuracy: null,
+        score: null,
+        retry: 3,
+        globalVariables: variables
+      }
+    }
+  );
+
+  await addEvaluationJob({ evalId: evaluation._id });
+}
+
+export default NextAPI(handler);

projects/app/src/pages/dashboard/evaluation/create.tsx

@@ -20,11 +20,11 @@ import { getAppDetailById } from '@/web/core/app/api';
 import { useToast } from '@fastgpt/web/hooks/useToast';
 import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import { fileDownload } from '@/web/common/file/utils';
-import { postCreateEvaluation } from '@/web/core/app/api/evaluation';
+import { postCreateEvaluation } from '@/web/core/evaluation/evaluation';
 import { useMemo, useState } from 'react';
 import Markdown from '@/components/Markdown';
-import { getEvaluationFileHeader } from '@fastgpt/global/core/app/evaluation/utils';
-import { evaluationFileErrors } from '@fastgpt/global/core/app/evaluation/constants';
+import { getEvaluationFileHeader } from '@fastgpt/global/core/evaluation/utils';
+import { evaluationFileErrors } from '@fastgpt/global/core/evaluation/constants';
 import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 

projects/app/src/pages/dashboard/evaluation/index.tsx

@@ -20,15 +20,15 @@ import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useRouter } from 'next/router';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
-import { deleteEvaluation, getEvaluationList } from '@/web/core/app/api/evaluation';
+import { deleteEvaluation, getEvaluationList } from '@/web/core/evaluation/evaluation';
 import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
 import Avatar from '@fastgpt/web/components/common/Avatar';
 import { usePagination } from '@fastgpt/web/hooks/usePagination';
 import { useState, useEffect, useMemo } from 'react';
-import EvaluationDetailModal from '../../../pageComponents/app/evaluation/DetailModal';
+import EvaluationDetailModal from '@/pageComponents/evaluation/DetailModal';
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
 import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
-import type { evaluationType } from '@fastgpt/global/core/app/evaluation/type';
+import type { evaluationType } from '@fastgpt/global/core/evaluation/type';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConfirm';
 

projects/app/src/web/core/evaluation/evaluation.ts

@@ -4,8 +4,8 @@ import type {
   listEvaluationsBody,
   retryEvalItemBody,
   updateEvalItemBody
-} from '@fastgpt/global/core/app/evaluation/api';
-import type { evaluationType, listEvalItemsItem } from '@fastgpt/global/core/app/evaluation/type';
+} from '@fastgpt/global/core/evaluation/api';
+import type { evaluationType, listEvalItemsItem } from '@fastgpt/global/core/evaluation/type';
 import type { PaginationResponse } from '@fastgpt/web/common/fetch/type';
 
 export const postCreateEvaluation = ({
@@ -25,7 +25,7 @@ export const postCreateEvaluation = ({
   formData.append('file', file, encodeURIComponent(file.name));
   formData.append('data', JSON.stringify({ name, evalModel, appId }));
 
-  return POST(`/proApi/core/app/evaluation/create`, formData, {
+  return POST(`/core/evaluation/create`, formData, {
     timeout: 600000,
     onUploadProgress: (e) => {
       if (!e.total) return;
@@ -40,19 +40,18 @@ export const postCreateEvaluation = ({
 };
 
 export const getEvaluationList = (data: listEvaluationsBody) =>
-  POST<PaginationResponse<evaluationType>>('/proApi/core/app/evaluation/list', data);
+  POST<PaginationResponse<evaluationType>>('/core/evaluation/list', data);
 
 export const deleteEvaluation = (data: { evalId: string }) =>
-  DELETE('/proApi/core/app/evaluation/delete', data);
+  DELETE('/core/evaluation/delete', data);
 
 export const getEvalItemsList = (data: listEvalItemsBody) =>
-  POST<PaginationResponse<listEvalItemsItem>>('/proApi/core/app/evaluation/listItems', data);
+  POST<PaginationResponse<listEvalItemsItem>>('/core/evaluation/listItems', data);
 
 export const deleteEvalItem = (data: { evalItemId: string }) =>
-  DELETE('/proApi/core/app/evaluation/deleteItem', data);
+  DELETE('/core/evaluation/deleteItem', data);
 
-export const retryEvalItem = (data: retryEvalItemBody) =>
-  POST('/proApi/core/app/evaluation/retryItem', data);
+export const retryEvalItem = (data: retryEvalItemBody) => POST('/core/evaluation/retryItem', data);
 
 export const updateEvalItem = (data: updateEvalItemBody) =>
-  POST('/proApi/core/app/evaluation/updateItem', data);
+  POST('/core/evaluation/updateItem', data);