2c681bcdd1
* fix: text split * remove test
6.2 KiB
[
"这是一个测试的内容，包含代码块快速了解FastGPTFastGPT的能力与优势FastGPT是一个基于LLM大语言模型的知识库问答系统，提供开箱即用的数据处理、模型调用等能力。同时可以通过Flow可视化进行工作流编排，从而实现复杂的问答场景！FastGPT在线使用：https://fastgpt.ioFastGPT能力1.专属AI客服通过导入文档或已有问答对进行训练，让AI模型能根据你的文档以交互式对话方式回答问题。2.简单易用的可视化界面FastGPT采用直观的可视化界面设计，为各种应用场景提供了丰富实用的功能。通过简洁易懂的操作步骤，可以轻松完成AI客服的创建和训练流程。jsimport{defaultMaxChunkSize}from'../../core/dataset/training/utils';import{getErrText}from'../error/utils';constgetOneTextOverlapText=({text,step}:{text:string;step:number}):string=>{constforbidOverlap=checkForbidOverlap(step);constmaxOverlapLen=chunkSize0.4;//step>=stepReges.length:Donotoverlapincompletesentencesif(forbidOverlap||overlapLen===0||step>=stepReges.length)return'';constsplitTexts=getSplitTexts({text,step});letoverlayText='';for(leti=splitTexts.length-1;i>=0;i--){constcurrentText=splitTexts[i].text;constnewText=currentText+overlayText;constnewTextLen=newText.length;if(newTextLen>overlapLen){if(newTextLen>maxOverlapLen){consttext=getOneTextOverlapText({text:newText,step:step+1});returntext||overlayText;}returnnewText;}overlayText=newText;}returnoverlayText;};constgetOneTextOverlapText=({text,step}:{text:string;step:number}):string=>{constforbidOverlap=checkForbidOverlap(step);constmaxOverlapLen=chunkSize0.4;//step>=stepReges.length:Donotoverlapincompletesentencesif(forbidOverlap||overlapLen===0||step>=stepReges.length)return'';constsplitTexts=getSplitTexts({text,step});letoverlayText='';for(leti=splitTexts.length-1;i>=0;i--){constcurrentText=splitTexts[i].text;constnewText=currentText+overlayText;constnewTextLen=newText.length;if(newTextLen>overlapLen){if(newTextLen>maxOverlapLen){consttext=getOneTextOverlapText({text:newText,step:step+1});returntext||overlayText;}returnnewText;}overlayText=newText;}returnoverlayText;};constgetOneTextOverlapText=({text,step}:{text:string;step:number}):string=>{constforbidOverlap=checkForbidOverlap(step);constmaxOverlapLen=chunkSize0.4;//step>=stepReges.length:Donotoverlapincompletesentencesif(forbidOverlap||overlapLen===0||step>=stepReges.length)return'';constsplitTexts=getSplitTexts({text,step});letoverlayText='';for(leti=splitTexts.length-1;i>=0;i--){constcurrentText=splitTexts[i].text;constnewText=currentText+overlayText;constnewTextLen=newText.length;if(newTextLen>overlapLen){if(newTextLen>maxOverlapLen){consttext=getOneTextOverlapText({text:newText,step:step+1});returntext||overlayText;}returnnewText;}overlayText=newText;}returnoverlayText;};constgetOneTextOverlapText=({text,step}:{text:string;step:number}):string=>{constforbidOverlap=checkForbidOverlap(step);constmaxOverlapLen=chunkSize0.4;//step>=stepReges.length:Donotoverlapincompletesentencesif(forbidOverlap||overlapLen===0||step>=stepReges.length)return'';constsplitTexts=getSplitTexts({text,step});letoverlayText='';for(leti=splitTexts.length-1;i>=0;i--){constcurrentText=splitTexts[i].text;constnewText=currentText+overlayText;constnewTextLen=newText.length;if(newTextLen>overlapLen){if(newTextLen>maxOverlapLen){consttext=getOneTextOverlapText({text:newText,step:step+1});returntext||overlayText;}returnnewText;}overlayText=newText;}returnoverlayText;};constgetOneTextOverlapText=({text,step}:{text:string;step:number}):string=>{constforbidOverlap=checkForbidOverlap(step);constmaxOverlapLen=chunkSize*0.4;//step>=stepReges.length:Donotoverlapincompletesentencesif(forbidOverlap||overlapLen===0||step>=stepReges.length)return'';constsplitTexts=getSplitTexts({text,step});letoverlayText='';for(leti=splitTexts.length-1;i>=0;i--){constcurrentText=splitTexts[i].text;constnewText=currentText+overlayText;constnewTextLen=newText.length;if(newTextLen>overlapLen){if(newTextLen>maxOverlapLen){consttext=getOneTextOverlapText({text:newText,step:step+1});returntext||overlayText;}returnnewText;}overlayText=newText;}returnoverlayText;};