From 16ab1f0eaea4819bf5b0bb4893143e7eb039f995 Mon Sep 17 00:00:00 2001
From: shaohuzhang1 <80892890+shaohuzhang1@users.noreply.github.com>
Date: Mon, 1 Apr 2024 14:39:56 +0800
Subject: [PATCH] Pr@main@fix bugs (#27)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix: 优化word分段规则

* fix: 去除标题特殊字符

* fix: 对话重新生成问题

---------

Co-authored-by: wangdan-fit2cloud <dan.wang@fit2cloud.com>
---
 apps/common/handle/impl/doc_split_handle.py      | 16 +++++++++++++++-
 apps/common/util/split_model.py                  |  8 +++++++-
 ui/src/api/application.ts                        | 10 ++++------
 ui/src/components/ai-chat/index.vue              | 12 ++++++++----
 ui/src/views/applicaiton-overview/index.vue      |  4 ++--
 .../views/dataset/component/UploadComponent.vue  |  2 +-
 6 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/apps/common/handle/impl/doc_split_handle.py b/apps/common/handle/impl/doc_split_handle.py
index 50addb32c..d3d5ee6c7 100644
--- a/apps/common/handle/impl/doc_split_handle.py
+++ b/apps/common/handle/impl/doc_split_handle.py
@@ -22,11 +22,25 @@ default_pattern_list = [re.compile('(?<=^)# .*|(?<=\\n)# .*'), re.compile('(?<!#
 
 
 class DocSplitHandle(BaseSplitHandle):
+    @staticmethod
+    def paragraph_to_md(paragraph):
+        psn = paragraph.style.name
+        if psn.startswith('Heading'):
+            try:
+                return "".join(["#" for i in range(int(psn.replace("Heading ", '')))]) + " " + paragraph.text
+            except Exception as e:
+                return paragraph.text
+        return paragraph.text
+
+    def to_md(self, doc):
+        ps = doc.paragraphs
+        return "\n".join([self.paragraph_to_md(para) for para in ps])
+
     def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer):
         try:
             buffer = get_buffer(file)
             doc = Document(io.BytesIO(buffer))
-            content = "\n".join([para.text for para in doc.paragraphs])
+            content = self.to_md(doc)
             if pattern_list is not None and len(pattern_list) > 0:
                 split_model = SplitModel(pattern_list, with_filter, limit)
             else:
diff --git a/apps/common/util/split_model.py b/apps/common/util/split_model.py
index 00184f3c1..60ddaac2f 100644
--- a/apps/common/util/split_model.py
+++ b/apps/common/util/split_model.py
@@ -351,9 +351,15 @@ class SplitModel:
 
     @staticmethod
     def filter_title_special_characters(paragraph: Dict):
-        return {**paragraph, 'title': paragraph.get('title').replace("#", '') if 'title' in paragraph else ''}
+        title = paragraph.get('title') if 'title' in paragraph else ''
+        for title_special_characters in title_special_characters_list:
+            title = title.replace(title_special_characters, '')
+        return {**paragraph,
+                'title': title}
 
 
+title_special_characters_list = ['#', '\n', '\r', '\\s']
+
 default_split_pattern = {
     'md': [re.compile('(?<=^)# .*|(?<=\\n)# .*'), re.compile('(?<!#)## (?!#).*'), re.compile("(?<!#)### (?!#).*"),
            re.compile("(?<!#)#### (?!#).*"), re.compile("(?<!#)##### (?!#).*"),
diff --git a/ui/src/api/application.ts b/ui/src/api/application.ts
index 1a3631d36..7cc99fabc 100644
--- a/ui/src/api/application.ts
+++ b/ui/src/api/application.ts
@@ -166,14 +166,12 @@ const getChatOpen: (applicaiton_id: String) => Promise<Result<any>> = (applicait
 }
 /**
  * 对话
- * @param 参数 
+ * @param 参数
  * chat_id: string
- * {
-    "message": "string",
-  }
+ * data
  */
-const postChatMessage: (chat_id: string, message: string) => Promise<any> = (chat_id, message) => {
-  return postStream(`/api${prefix}/chat_message/${chat_id}`, { message })
+const postChatMessage: (chat_id: string, data: any) => Promise<any> = (chat_id, message) => {
+  return postStream(`/api${prefix}/chat_message/${chat_id}`, data)
 }
 
 /**
diff --git a/ui/src/components/ai-chat/index.vue b/ui/src/components/ai-chat/index.vue
index 1330f4814..2d3d4d753 100644
--- a/ui/src/components/ai-chat/index.vue
+++ b/ui/src/components/ai-chat/index.vue
@@ -285,7 +285,7 @@ function sendChatHandle(event: any) {
   if (!event.ctrlKey) {
     // 如果没有按下组合键ctrl，则会阻止默认事件
     event.preventDefault()
-    if (!isDisabledChart.value && !loading.value&&!event.isComposing) {
+    if (!isDisabledChart.value && !loading.value && !event.isComposing) {
       chatMessage()
     }
   } else {
@@ -418,7 +418,7 @@ const errorWrite = (chat: any, message?: string) => {
   ChatManagement.append(chat.id, message || '抱歉，当前正在维护，无法提供服务，请稍后再试！')
   ChatManagement.close(chat.id)
 }
-function chatMessage(chat?: any, problem?: string) {
+function chatMessage(chat?: any, problem?: string, re_chat?: boolean) {
   loading.value = true
   if (!chat) {
     chat = reactive({
@@ -443,9 +443,13 @@ function chatMessage(chat?: any, problem?: string) {
       errorWrite(chat)
     })
   } else {
+    const obj = {
+      message: chat.problem_text,
+      re_chat: re_chat || false
+    }
     // 对话
     applicationApi
-      .postChatMessage(chartOpenId.value, chat.problem_text)
+      .postChatMessage(chartOpenId.value, obj)
       .then((response) => {
         if (response.status === 401) {
           application
@@ -491,7 +495,7 @@ function chatMessage(chat?: any, problem?: string) {
 
 function regenerationChart(item: chatType) {
   inputValue.value = item.problem_text
-  chatMessage()
+  chatMessage(null, '', true)
 }
 
 function getSourceDetail(row: any) {
diff --git a/ui/src/views/applicaiton-overview/index.vue b/ui/src/views/applicaiton-overview/index.vue
index 6b0d82561..d981e5063 100644
--- a/ui/src/views/applicaiton-overview/index.vue
+++ b/ui/src/views/applicaiton-overview/index.vue
@@ -197,8 +197,8 @@ function getAppStatistics() {
 
 function refreshAccessToken() {
   MsgConfirm(
-    `是否重新生成公共访问链接?`,
-    `重新生成公共访问链接会影响嵌入第三方脚本变更，需要将新脚本重新嵌入第三方，请谨慎操作！`,
+    `是否重新生成公开访问链接?`,
+    `重新生成公开访问链接会影响嵌入第三方脚本变更，需要将新脚本重新嵌入第三方，请谨慎操作！`,
     {
       confirmButtonText: '确认'
     }
diff --git a/ui/src/views/dataset/component/UploadComponent.vue b/ui/src/views/dataset/component/UploadComponent.vue
index b750889c5..aadac6b3c 100644
--- a/ui/src/views/dataset/component/UploadComponent.vue
+++ b/ui/src/views/dataset/component/UploadComponent.vue
@@ -27,7 +27,7 @@
             <em> 选择文件上传 </em>
           </p>
           <div class="upload__decoration">
-            <p>支持格式：TXT、Markdown，每次最多上传50个文件，每个文件不超过 10MB</p>
+            <p>支持格式：TXT、Markdown、PDF、DOC、DOCX，每次最多上传50个文件，每个文件不超过 10MB</p>
             <p>若使用【高级分段】建议上传前规范文件的分段标识</p>
           </div>
         </div>