From 8b9998a53def3796e3ab059ac81e155e763f3436 Mon Sep 17 00:00:00 2001
From: shaohuzhang1 <80892890+shaohuzhang1@users.noreply.github.com>
Date: Fri, 7 Mar 2025 10:19:57 +0800
Subject: [PATCH] fix: When importing documents from a web knowledge base, if a
 failed document is encountered, the import will be interrupted (#2511)

---
 apps/dataset/task/sync.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/apps/dataset/task/sync.py b/apps/dataset/task/sync.py
index 7b5c929ba..16add2db1 100644
--- a/apps/dataset/task/sync.py
+++ b/apps/dataset/task/sync.py
@@ -54,5 +54,8 @@ def sync_replace_web_dataset(dataset_id: str, url: str, selector: str):
 def sync_web_document(dataset_id, source_url_list: List[str], selector: str):
     handler = get_sync_web_document_handler(dataset_id)
     for source_url in source_url_list:
-        result = Fork(base_fork_url=source_url, selector_list=selector.split(' ')).fork()
-        handler(source_url, selector, result)
+        try:
+            result = Fork(base_fork_url=source_url, selector_list=selector.split(' ')).fork()
+            handler(source_url, selector, result)
+        except Exception as e:
+            pass