#!/usr/bin/env python3 """ 测试 Gemini 3 Pro 模型解析数学试卷的能力 对比 Claude Sonnet 4.5 和 Gemini 3 Pro 在数学试卷解析上的表现 """ import os import sys import json import time import logging from pathlib import Path from dotenv import load_dotenv from claude_exam_parser import ClaudeExamParser # 配置日志 logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) logger = logging.getLogger(__name__) # 加载环境变量 load_dotenv() def test_single_model(file_path: str, model: str, subject: str = "数学", grade: str = None): """ 使用单个模型测试试卷解析 Args: file_path: 试卷文件路径 model: 模型名称 subject: 学科 grade: 年级 Returns: 解析结果和性能指标 """ logger.info(f"\n{'='*80}") logger.info(f"开始测试模型: {model}") logger.info(f"文件: {file_path}") logger.info(f"学科: {subject}") logger.info(f"{'='*80}\n") try: # 创建解析器 parser = ClaudeExamParser(model=model) # 开始计时 start_time = time.time() # 解析试卷 result = parser.parse_exam_file(file_path, subject=subject, grade=grade) # 结束计时 end_time = time.time() elapsed_time = end_time - start_time if result["success"]: parsed_data = result["result"] logger.info(f"\n✅ 解析成功！") logger.info(f" 耗时: {elapsed_time:.2f} 秒") logger.info(f" 学科: {parsed_data.get('subject', 'N/A')}") logger.info(f" 年级: {parsed_data.get('grade', 'N/A')}") logger.info(f" 题目总数: {parsed_data.get('total_questions', 0)}") # 题型统计 stats = parsed_data.get("stats", {}) logger.info(f"\n 📊 题型统计:") logger.info(f" 选择题: {stats.get('choice_count', 0)}") logger.info(f" 填空题: {stats.get('fill_blank_count', 0)}") logger.info(f" 计算题: {stats.get('calculation_count', 0)}") logger.info(f" 解答题: {stats.get('essay_count', 0)}") # 难度分布（如果有） if "difficulty_distribution" in stats: logger.info(f"\n 📈 难度分布:") for level, count in stats["difficulty_distribution"].items(): logger.info(f" 难度 {level}: {count} 题") # 分析公式数量 formula_count = 0 questions_with_formulas = 0 for q in parsed_data.get("questions", []): formulas = q.get("formulas", []) if formulas: formula_count += len(formulas) questions_with_formulas += 1 logger.info(f"\n 📐 公式统计:") logger.info(f" 总公式数: {formula_count}") logger.info(f" 包含公式的题目: {questions_with_formulas}/{parsed_data.get('total_questions', 0)}") # 显示前3道题作为示例 logger.info(f"\n 📝 前3道题目示例:") for i, q in enumerate(parsed_data.get("questions", [])[:3], 1): logger.info(f"\n 题目 {q.get('number', i)}:") logger.info(f" 类型: {q.get('type', 'N/A')}") logger.info(f" 题干: {q.get('question', 'N/A')[:100]}...") if q.get("formulas"): logger.info(f" 公式: {q.get('formulas', [])[:2]}") if q.get("has_image"): logger.info(f" 图片: {q.get('image_description', 'N/A')[:80]}...") logger.info(f" 难度: {q.get('difficulty', 'N/A')}") logger.info(f" 知识点: {q.get('knowledge_points', [])}") return { "success": True, "model": model, "elapsed_time": elapsed_time, "total_questions": parsed_data.get('total_questions', 0), "formula_count": formula_count, "questions_with_formulas": questions_with_formulas, "stats": stats, "result": parsed_data } else: logger.error(f"\n❌ 解析失败: {result.get('error', 'Unknown error')}") return { "success": False, "model": model, "error": result.get('error', 'Unknown error') } except Exception as e: logger.error(f"\n❌ 测试失败: {str(e)}", exc_info=True) return { "success": False, "model": model, "error": str(e) } def compare_models(file_path: str, subject: str = "数学", grade: str = None): """ 对比多个模型的解析效果 Args: file_path: 试卷文件路径 subject: 学科 grade: 年级 """ models = [ "anthropic/claude-sonnet-4.5", "google/gemini-3-pro-preview" ] results = {} for model in models: results[model] = test_single_model(file_path, model, subject, grade) # 生成对比报告 logger.info(f"\n\n{'='*80}") logger.info(f"对比报告") logger.info(f"{'='*80}\n") comparison_table = [] for model, result in results.items(): if result["success"]: comparison_table.append({ "模型": model, "耗时(秒)": f"{result['elapsed_time']:.2f}", "题目数": result['total_questions'], "公式数": result['formula_count'], "含公式题目": f"{result['questions_with_formulas']}/{result['total_questions']}", "选择题": result['stats'].get('choice_count', 0), "填空题": result['stats'].get('fill_blank_count', 0), "计算题": result['stats'].get('calculation_count', 0), "解答题": result['stats'].get('essay_count', 0), }) else: comparison_table.append({ "模型": model, "状态": f"❌ 失败: {result.get('error', 'Unknown')}", }) # 打印对比表格 logger.info("对比结果:") logger.info(json.dumps(comparison_table, ensure_ascii=False, indent=2)) # 保存详细结果到文件 output_dir = Path("test_results") output_dir.mkdir(exist_ok=True) timestamp = time.strftime("%Y%m%d_%H%M%S") for model, result in results.items(): if result["success"]: model_name = model.replace("/", "_") output_file = output_dir / f"result_{model_name}_{timestamp}.json" with open(output_file, "w", encoding="utf-8") as f: json.dump(result["result"], f, ensure_ascii=False, indent=2) logger.info(f"\n💾 {model} 详细结果已保存到: {output_file}") logger.info(f"\n{'='*80}\n") def main(): """主函数""" if len(sys.argv) < 2: print("用法:") print(f" {sys.argv[0]} <试卷文件路径> [学科] [年级]") print() print("示例:") print(f" {sys.argv[0]} math_exam.pdf 数学 高一") print(f" {sys.argv[0]} physics_test.pdf 物理") print() print("环境变量:") print(" OPENROUTER_API_KEY - OpenRouter API Key（必需）") print() print("支持的模型:") print(" - anthropic/claude-sonnet-4.5") print(" - google/gemini-3-pro-preview") sys.exit(1) file_path = sys.argv[1] subject = sys.argv[2] if len(sys.argv) > 2 else "数学" grade = sys.argv[3] if len(sys.argv) > 3 else None # 检查文件是否存在 if not os.path.exists(file_path): logger.error(f"文件不存在: {file_path}") sys.exit(1) # 检查 API Key if not os.getenv("OPENROUTER_API_KEY"): logger.error("未设置 OPENROUTER_API_KEY 环境变量") logger.error("请在 .env 文件中设置或使用: export OPENROUTER_API_KEY='your-key'") sys.exit(1) # 运行对比测试 compare_models(file_path, subject, grade) if __name__ == "__main__": main()