From ef66af639e00c0e968de94068f4c632c24a667f4 Mon Sep 17 00:00:00 2001
From: shaohuzhang1 <shaohu.zhang@fit2cloud.com>
Date: Wed, 24 Jan 2024 11:23:16 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20=E5=90=8C=E6=AD=A5web=E7=AB=99=E7=82=B9?=
 =?UTF-8?q?=20=20=E9=80=89=E6=8B=A9=E5=99=A8=E4=B8=8D=E5=A1=AB=E6=97=A0?=
 =?UTF-8?q?=E6=B3=95=E8=8E=B7=E5=8F=96=E5=86=85=E5=AE=B9?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 apps/common/event/listener_manage.py             | 4 ++--
 apps/common/util/fork.py                         | 2 +-
 apps/dataset/serializers/document_serializers.py | 3 ++-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/apps/common/event/listener_manage.py b/apps/common/event/listener_manage.py
index 14b371bfd..158f1cd67 100644
--- a/apps/common/event/listener_manage.py
+++ b/apps/common/event/listener_manage.py
@@ -181,8 +181,8 @@ class ListenerManagement:
     def sync_web_dataset(args: SyncWebDatasetArgs):
         if try_lock('sync_web_dataset' + args.lock_key):
             try:
-                ForkManage(args.url, args.selector.split(" ")).fork(2, set(),
-                                                                    args.handler)
+                ForkManage(args.url, args.selector.split(" ") if args.selector is not None else []).fork(2, set(),
+                                                                                                         args.handler)
             except Exception as e:
                 logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}')
             finally:
diff --git a/apps/common/util/fork.py b/apps/common/util/fork.py
index 8897d6dee..6c70ce796 100644
--- a/apps/common/util/fork.py
+++ b/apps/common/util/fork.py
@@ -59,7 +59,7 @@ class Fork:
     def __init__(self, base_fork_url: str, selector_list: List[str]):
         self.base_fork_url = urljoin(base_fork_url if base_fork_url.endswith("/") else base_fork_url + '/', '.')
         self.base_fork_url = self.base_fork_url[:-1]
-        self.selector_list = selector_list
+        self.selector_list = [selector for selector in selector_list if selector is not None and len(selector) > 0]
         self.urlparse = urlparse(self.base_fork_url)
         self.base_url = ParseResult(scheme=self.urlparse.scheme, netloc=self.urlparse.netloc, path='', params='',
                                     query='',
diff --git a/apps/dataset/serializers/document_serializers.py b/apps/dataset/serializers/document_serializers.py
index 400964c1b..949fe0b2f 100644
--- a/apps/dataset/serializers/document_serializers.py
+++ b/apps/dataset/serializers/document_serializers.py
@@ -166,7 +166,8 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
                 document.status = Status.embedding
                 document.save()
                 source_url = document.meta.get('source_url')
-                selector_list = document.meta.get('selector').split(" ") if 'selector' in document.meta else []
+                selector_list = document.meta.get('selector').split(
+                    " ") if 'selector' in document.meta and document.meta.get('selector') is not None else []
                 result = Fork(source_url, selector_list).fork()
                 if result.status == 200:
                     # 删除段落