gaoandQwen-Coder
1701472642
- Implemented enhanced PDF parsing with V2 parser for better question extraction - Added improved question formatting and display components - Integrated question navigation and progress tracking features - Added JSON export functionality for paper questions - Enhanced image extraction and association with questions - Improved database configuration with environment variables - Updated UI with responsive question cards and navigation sidebar - Added mathematical symbol rendering and question type detection Co-authored-by: Qwen-Coder <[email protected]>
220 lines
9.3 KiB
Python
220 lines
9.3 KiB
Python
"""
|
||
混合PDF拆分器：文本层解析 + OCR备份
|
||
优先使用文本层解析（快速准确），OCR作为扫描版PDF的备份方案
|
||
"""
|
||
|
||
import os
|
||
import time
|
||
from typing import Dict, List
|
||
from pdf_exam_parser_enhanced import PDFExamParserEnhanced
|
||
from pdf_exam_parser_enhanced_v2 import PDFExamParserEnhancedV2
|
||
from pdf_exam_parser_fixed import PDFExamParserFixed
|
||
from pdf_splitter_improved import PDFSplitterImproved
|
||
from pdf_image_extractor import PDFImageExtractor
|
||
|
||
class PDFSplitterHybrid:
|
||
def __init__(self):
|
||
self.text_parser = PDFExamParserEnhanced()
|
||
self.enhanced_parser_v2 = PDFExamParserEnhancedV2() # 新增V2解析器
|
||
self.fixed_parser = PDFExamParserFixed()
|
||
self.ocr_parser = PDFSplitterImproved()
|
||
self.image_extractor = PDFImageExtractor()
|
||
|
||
def has_extractable_text(self, pdf_path: str) -> bool:
|
||
"""
|
||
检查PDF是否包含可提取的文本层
|
||
"""
|
||
try:
|
||
import fitz
|
||
doc = fitz.open(pdf_path)
|
||
|
||
# 检查前几页是否有文本
|
||
text_found = False
|
||
max_pages_to_check = min(3, len(doc))
|
||
|
||
for page_num in range(max_pages_to_check):
|
||
page = doc[page_num]
|
||
text = page.get_text().strip()
|
||
|
||
# 如果找到足够的文本内容，认为有文本层
|
||
if len(text) > 100: # 至少100个字符
|
||
text_found = True
|
||
break
|
||
|
||
doc.close()
|
||
return text_found
|
||
|
||
except Exception as e:
|
||
print(f"⚠️ 检查文本层失败: {e}")
|
||
return False
|
||
|
||
def fix_empty_options(self, pdf_path: str, result: Dict) -> Dict:
|
||
"""
|
||
修复空选项问题
|
||
"""
|
||
try:
|
||
import fitz
|
||
doc = fitz.open(pdf_path)
|
||
|
||
for question in result.get("questions", []):
|
||
if question.get("type") == "single_choice" and question.get("options"):
|
||
options = question["options"]
|
||
empty_options = [k for k, v in options.items() if v.strip() == ""]
|
||
|
||
if empty_options:
|
||
# 提取该题目页面的文本
|
||
page_num = question.get("page", 1) - 1
|
||
if page_num < len(doc):
|
||
page = doc[page_num]
|
||
text = page.get_text()
|
||
|
||
# 查找选项模式
|
||
for opt_label in empty_options:
|
||
pattern = f"{opt_label}\\s*[.．)]\\s*([^A-D]*?)(?=[A-D]\\s*[.．)]|$)"
|
||
import re
|
||
match = re.search(pattern, text, re.DOTALL)
|
||
if match:
|
||
content = match.group(1).strip()
|
||
if content:
|
||
options[opt_label] = content
|
||
print(f"🔧 修复选项{opt_label}: {content[:30]}...")
|
||
else:
|
||
# 备用方案：标记为数学符号
|
||
options[opt_label] = f"[数学符号-{opt_label}]"
|
||
print(f"⚠️ 无法修复选项{opt_label}，标记为数学符号")
|
||
|
||
doc.close()
|
||
|
||
except Exception as e:
|
||
print(f"⚠️ 选项修复失败: {e}")
|
||
|
||
return result
|
||
|
||
def process_pdf(self, pdf_path: str, output_dir: str = None) -> Dict:
|
||
"""
|
||
混合处理策略：优先文本层，后备OCR
|
||
"""
|
||
total_start = time.time()
|
||
print(f"🎯 开始混合PDF处理: {pdf_path}")
|
||
|
||
try:
|
||
# 1. 检查是否有可提取的文本层
|
||
has_text = self.has_extractable_text(pdf_path)
|
||
|
||
if has_text:
|
||
print("✅ 检测到文本层，优先使用V2增强解析器")
|
||
result = self.enhanced_parser_v2.process_pdf(pdf_path)
|
||
|
||
# 验证V2解析结果质量
|
||
if (result.get("success") and
|
||
result.get("stats", {}).get("total_questions", 0) > 0):
|
||
|
||
print(f"🎉 V2增强解析成功: {result['stats']['total_questions']}道题")
|
||
result["stats"]["method"] = "enhanced_v2_parsing"
|
||
|
||
# 如果V2解析结果不理想，回退到原解析器
|
||
if result["stats"]["total_questions"] < 2:
|
||
print("⚠️ V2解析题目数量较少，回退到原增强解析器")
|
||
fallback_result = self.text_parser.process_pdf(pdf_path)
|
||
if fallback_result.get("success") and fallback_result.get("questions_found", 0) > result["stats"]["total_questions"]:
|
||
result = fallback_result
|
||
result["stats"]["method"] = "enhanced_text_fallback"
|
||
print(f"🔄 回退成功: {result['questions_found']}道题")
|
||
|
||
# 标准化数据格式
|
||
if "questions" in result and "stats" in result:
|
||
# V2解析器已经有标准格式
|
||
questions_count = result["stats"]["total_questions"]
|
||
else:
|
||
# 旧格式需要适配
|
||
result["questions"] = result.get("questions", [])
|
||
questions_count = len(result["questions"])
|
||
result["stats"] = {"total_questions": questions_count, "method": "enhanced_text_parsing"}
|
||
|
||
# 检查选择题选项是否有问题
|
||
choice_questions = [q for q in result.get("questions", []) if q.get("type") == "single_choice"]
|
||
has_empty_options = any(
|
||
any(opt.strip() == "" for opt in (q.get("options", {}).values()))
|
||
for q in choice_questions
|
||
)
|
||
|
||
if has_empty_options:
|
||
print("⚠️ 检测到选择题选项有空值，尝试选项修复")
|
||
# 尝试修复空选项
|
||
result = self.fix_empty_options(pdf_path, result)
|
||
result["stats"]["method"] = result["stats"]["method"] + "_with_fix"
|
||
|
||
# 提取并关联图片 - 始终执行
|
||
print("🖼️ 开始提取PDF图片...")
|
||
try:
|
||
updated_questions, extracted_images = self.image_extractor.extract_and_associate(
|
||
pdf_path, result["questions"]
|
||
)
|
||
result["questions"] = updated_questions
|
||
result["stats"]["images_extracted"] = len(extracted_images)
|
||
print(f"✅ 图片提取完成: {len(extracted_images)}张图片")
|
||
except Exception as e:
|
||
print(f"⚠️ 图片提取失败: {e}")
|
||
result["stats"]["images_extracted"] = 0
|
||
|
||
return result
|
||
else:
|
||
print("⚠️ 增强文本层解析结果不理想，回退到OCR")
|
||
else:
|
||
print("⚠️ 未检测到文本层，直接使用OCR")
|
||
|
||
# 2. 使用OCR作为备份或主要方案
|
||
print("🔄 使用OCR解析")
|
||
result = self.ocr_parser.process_pdf(pdf_path, output_dir)
|
||
|
||
if result.get("success"):
|
||
result["stats"]["method"] = "ocr_backup" if has_text else "ocr_primary"
|
||
print(f"✅ OCR解析完成: {result.get('questions_found', 0)}道题")
|
||
|
||
return result
|
||
|
||
except Exception as e:
|
||
print(f"❌ 混合处理失败: {str(e)}")
|
||
return {"success": False, "error": str(e)}
|
||
|
||
# 测试函数
|
||
def test_hybrid_parser():
|
||
"""
|
||
测试混合解析器
|
||
"""
|
||
pdf_path = "/Users/brunogao/Downloads/深圳市八年级（上）期中数学试卷1.pdf"
|
||
|
||
if not os.path.exists(pdf_path):
|
||
print(f"❌ 测试文件不存在: {pdf_path}")
|
||
return
|
||
|
||
parser = PDFSplitterHybrid()
|
||
result = parser.process_pdf(pdf_path)
|
||
|
||
if result["success"]:
|
||
print(f"""
|
||
📊 混合解析测试结果:
|
||
- 总时间: {result['total_time']:.2f}秒
|
||
- 页数: {result['pages_processed']}
|
||
- 题目: {result['questions_found']}道
|
||
- 方法: {result['stats']['method']}
|
||
""")
|
||
|
||
print("前3道选择题详细:")
|
||
choice_count = 0
|
||
for q in result['questions']:
|
||
if q.get('type') == 'single_choice' and choice_count < 3:
|
||
choice_count += 1
|
||
print(f"\\n题目{q['number']}:")
|
||
print(f"题干: {q['text']}")
|
||
if q.get('options'):
|
||
print("选项:")
|
||
for k, v in q['options'].items():
|
||
print(f" {k}. {v}")
|
||
elif choice_count >= 3:
|
||
break
|
||
else:
|
||
print(f"❌ 测试失败: {result['error']}")
|
||
|
||
if __name__ == "__main__":
|
||
test_hybrid_parser() |