diff --git a/packages/service/core/dataset/search/controller.ts b/packages/service/core/dataset/search/controller.ts index 9e87bf9bc..9f59aba8d 100644 --- a/packages/service/core/dataset/search/controller.ts +++ b/packages/service/core/dataset/search/controller.ts @@ -134,12 +134,10 @@ export const filterDatasetDataByMaxTokens = async ( let totalTokens = 0; for await (const item of tokensScoreFilter) { + results.push(item); + totalTokens += item.tokens; - if (totalTokens > maxTokens + 500) { - break; - } - results.push(item); if (totalTokens > maxTokens) { break; }