diff --git a/apps/common/event/listener_manage.py b/apps/common/event/listener_manage.py index 14b371bfd..158f1cd67 100644 --- a/apps/common/event/listener_manage.py +++ b/apps/common/event/listener_manage.py @@ -181,8 +181,8 @@ class ListenerManagement: def sync_web_dataset(args: SyncWebDatasetArgs): if try_lock('sync_web_dataset' + args.lock_key): try: - ForkManage(args.url, args.selector.split(" ")).fork(2, set(), - args.handler) + ForkManage(args.url, args.selector.split(" ") if args.selector is not None else []).fork(2, set(), + args.handler) except Exception as e: logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}') finally: diff --git a/apps/common/util/fork.py b/apps/common/util/fork.py index 8897d6dee..6c70ce796 100644 --- a/apps/common/util/fork.py +++ b/apps/common/util/fork.py @@ -59,7 +59,7 @@ class Fork: def __init__(self, base_fork_url: str, selector_list: List[str]): self.base_fork_url = urljoin(base_fork_url if base_fork_url.endswith("/") else base_fork_url + '/', '.') self.base_fork_url = self.base_fork_url[:-1] - self.selector_list = selector_list + self.selector_list = [selector for selector in selector_list if selector is not None and len(selector) > 0] self.urlparse = urlparse(self.base_fork_url) self.base_url = ParseResult(scheme=self.urlparse.scheme, netloc=self.urlparse.netloc, path='', params='', query='', diff --git a/apps/dataset/serializers/document_serializers.py b/apps/dataset/serializers/document_serializers.py index 400964c1b..949fe0b2f 100644 --- a/apps/dataset/serializers/document_serializers.py +++ b/apps/dataset/serializers/document_serializers.py @@ -166,7 +166,8 @@ class DocumentSerializers(ApiMixin, serializers.Serializer): document.status = Status.embedding document.save() source_url = document.meta.get('source_url') - selector_list = document.meta.get('selector').split(" ") if 'selector' in document.meta else [] + selector_list = document.meta.get('selector').split( + " ") if 'selector' in document.meta and document.meta.get('selector') is not None else [] result = Fork(source_url, selector_list).fork() if result.status == 200: # 删除段落