From 9a11937a6be877bb77fea24af58ec9918a91aeec Mon Sep 17 00:00:00 2001
From: shaohuzhang1 <shaohu.zhang@fit2cloud.com>
Date: Wed, 3 Jan 2024 15:40:37 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E5=90=8C=E6=AD=A5=E6=96=87=E6=A1=A3?=
 =?UTF-8?q?=E6=94=B9=E4=B8=BA=E5=BC=82=E6=AD=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 apps/dataset/serializers/document_serializers.py | 14 ++++++++++----
 apps/dataset/views/document.py                   |  1 +
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/apps/dataset/serializers/document_serializers.py b/apps/dataset/serializers/document_serializers.py
index bc0a4363e..7665b4c82 100644
--- a/apps/dataset/serializers/document_serializers.py
+++ b/apps/dataset/serializers/document_serializers.py
@@ -20,6 +20,7 @@ from drf_yasg import openapi
 from rest_framework import serializers
 
 from common.db.search import native_search, native_page_search
+from common.event.common import work_thread_pool
 from common.event.listener_manage import ListenerManagement
 from common.exception.app_exception import AppApiException
 from common.mixins.api_mixin import ApiMixin
@@ -115,7 +116,7 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
             if first.type != Type.web:
                 raise AppApiException(500, "只有web站点类型才支持同步")
 
-        def sync(self, with_valid=True):
+        def sync(self, with_valid=True, with_embedding=True):
             if with_valid:
                 self.is_valid(raise_exception=True)
             document_id = self.data.get('document_id')
@@ -146,6 +147,9 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
                     QuerySet(Paragraph).bulk_create(paragraph_model_list) if len(paragraph_model_list) > 0 else None
                     # 批量插入问题
                     QuerySet(Problem).bulk_create(problem_model_list) if len(problem_model_list) > 0 else None
+                    # 向量化
+                    if with_embedding:
+                        ListenerManagement.embedding_by_document_signal.send(document_id)
                 else:
                     document.status = Status.error
                     document.save()
@@ -203,10 +207,12 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
             document_id = self.data.get("document_id")
             document = QuerySet(Document).filter(id=document_id).first()
             if document.type == Type.web:
-                # 如果是web站点,就是先同步
-                DocumentSerializers.Sync(data={'document_id': document_id}).sync()
+                # 异步同步
+                work_thread_pool.submit(lambda x: DocumentSerializers.Sync(data={'document_id': document_id}).sync(),
+                                        {})
 
-            ListenerManagement.embedding_by_document_signal.send(document_id)
+            else:
+                ListenerManagement.embedding_by_document_signal.send(document_id)
             return True
 
         @transaction.atomic
diff --git a/apps/dataset/views/document.py b/apps/dataset/views/document.py
index 77418d118..cf5ca397e 100644
--- a/apps/dataset/views/document.py
+++ b/apps/dataset/views/document.py
@@ -15,6 +15,7 @@ from rest_framework.views import Request
 
 from common.auth import TokenAuth, has_permissions
 from common.constants.permission_constants import Permission, Group, Operate, PermissionConstants
+from common.event.common import work_thread_pool
 from common.response import result
 from common.util.common import query_params_to_single_dict
 from dataset.serializers.document_serializers import DocumentSerializers