15308555518/ MaxKB
1
0
Fork 0
mirror of https://github.com/1Panel-dev/MaxKB.git synced 2025-12-26 01:33:05 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

refactor: 使用lazy_load方式加载pdf
Some checks are pending
sync2gitee / repo-sync (push) Waiting to run
Details
Typos Check / Spell Check with Typos (push) Waiting to run
Details

This commit is contained in:
CaptainB 2024-08-16 10:36:27 +08:00 committed by 刘瑞斌
parent c332a6cacc
commit 0d59ab2be9
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/common/handle/impl/pdf_split_handle.py
View File

@ -26,10 +26,6 @@ default_pattern_list = [re.compile('(?<=^)# .*|(?<=\\n)# .*'),
re.compile("(?<!\n)\n\n+")]
def number_to_text(pdf_document, page_number):
return pdf_document[page_number].page_content
def check_pdf_is_image(pdf_path):
try:
# 打开PDF文件
@ -64,8 +60,9 @@ class PdfSplitHandle(BaseSplitHandle):
loader = PyPDFLoader(temp_file_path, extract_images=True)
else:
loader = PyPDFLoader(temp_file_path, extract_images=False)
pdf_document = loader.load()
content = "\n".join([number_to_text(pdf_document, page_number) for page_number in range(len(pdf_document))])
content = "\n".join([page.page_content for page in loader.lazy_load()])
if pattern_list is not None and len(pattern_list) > 0:
split_model = SplitModel(pattern_list, with_filter, limit)
else: