From d732a46f89b044599612e9ff84b7cdfb7f24ccf0 Mon Sep 17 00:00:00 2001
From: shaohuzhang1 <shaohu.zhang@fit2cloud.com>
Date: Mon, 25 Mar 2024 11:05:15 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20=E3=80=90=E7=9F=A5=E8=AF=86=E5=BA=93?=
 =?UTF-8?q?=E3=80=91=E5=AF=BC=E5=85=A5=E9=9D=9Eutf8=20=E7=BC=96=E7=A0=81?=
 =?UTF-8?q?=E7=9A=84txt=E6=96=87=E4=BB=B6=EF=BC=8C=E5=88=86=E6=AE=B5?=
 =?UTF-8?q?=E5=86=85=E5=AE=B9=E6=98=AF=E7=A9=BA=E7=99=BD?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 apps/common/util/split_model.py                  | 4 ++--
 apps/dataset/serializers/document_serializers.py | 3 ++-
 pyproject.toml                                   | 1 +
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/apps/common/util/split_model.py b/apps/common/util/split_model.py
index 93119647d..00184f3c1 100644
--- a/apps/common/util/split_model.py
+++ b/apps/common/util/split_model.py
@@ -357,7 +357,7 @@ class SplitModel:
 default_split_pattern = {
     'md': [re.compile('(?<=^)# .*|(?<=\\n)# .*'), re.compile('(?<!#)## (?!#).*'), re.compile("(?<!#)### (?!#).*"),
            re.compile("(?<!#)#### (?!#).*"), re.compile("(?<!#)##### (?!#).*"),
-           re.compile("(?<!#)###### (?!#).*")],
+           re.compile("(?<!#)###### (?!#).*"), re.compile("(?<!\n)\n\n+")],
     'default': [re.compile("(?<!\n)\n\n+")]
 }
 
@@ -374,7 +374,7 @@ def get_split_model(filename: str, with_filter: bool = False, limit: int = 4096)
         pattern_list = default_split_pattern.get('md')
         return SplitModel(pattern_list, with_filter=with_filter, limit=limit)
 
-    pattern_list = default_split_pattern.get('default')
+    pattern_list = default_split_pattern.get('md')
     return SplitModel(pattern_list, with_filter=with_filter, limit=limit)
 
 
diff --git a/apps/dataset/serializers/document_serializers.py b/apps/dataset/serializers/document_serializers.py
index 62bff658e..529479927 100644
--- a/apps/dataset/serializers/document_serializers.py
+++ b/apps/dataset/serializers/document_serializers.py
@@ -32,6 +32,7 @@ from dataset.models.data_set import DataSet, Document, Paragraph, Problem, Type,
 from dataset.serializers.common_serializers import BatchSerializer, MetaSerializer
 from dataset.serializers.paragraph_serializers import ParagraphSerializers, ParagraphInstanceSerializer
 from smartdoc.conf import PROJECT_DIR
+import chardet
 
 
 class DocumentEditInstanceSerializer(ApiMixin, serializers.Serializer):
@@ -599,7 +600,7 @@ def file_to_paragraph(file, pattern_list: List, with_filter: bool, limit: int):
     else:
         split_model = get_split_model(file.name, with_filter=with_filter, limit=limit)
     try:
-        content = data.decode('utf-8')
+        content = data.decode(chardet.detect(data)['encoding'])
     except BaseException as e:
         return {'name': file.name,
                 'content': []}
diff --git a/pyproject.toml b/pyproject.toml
index 268c23c96..d9a2f62b1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,6 +30,7 @@ html2text = "^2024.2.26"
 langchain-openai = "^0.0.8"
 django-ipware = "^6.0.4"
 django-apscheduler = "^0.6.2"
+chardet2 = "^2.0.3"
 
 [build-system]
 requires = ["poetry-core"]