lambdagent/agentexample/qaagent67/scripts/extract_pdfs.py

#!/usr/bin/env python3
"""批量提取 PDF 为文本文件，存到 processed/ 目录"""
import os
import sys
import fitz  # PyMuPDF
from pathlib import Path

RAW_DIR = Path('/home/67/knowledge/maritime/raw')
OUT_DIR = Path('/home/67/knowledge/maritime/processed')
OUT_DIR.mkdir(exist_ok=True)

pdf_files = sorted(RAW_DIR.glob('*.pdf'))
total = len(pdf_files)
print(f'共 {total} 个 PDF 文件')

success = 0
failed = 0
skipped = 0

for i, pdf_path in enumerate(pdf_files):
    out_path = OUT_DIR / (pdf_path.stem + '.txt')
    
    # 跳过已处理的
    if out_path.exists() and out_path.stat().st_size > 0:
        skipped += 1
        continue
    
    try:
        doc = fitz.open(str(pdf_path))
        text_parts = []
        for page_num, page in enumerate(doc):
            text = page.get_text()
            if text.strip():
                text_parts.append(f'--- 第{page_num+1}页 ---\n{text}')
        doc.close()
        
        full_text = '\n'.join(text_parts)
        if full_text.strip():
            with open(out_path, 'w', encoding='utf-8') as f:
                f.write(f'# 源文件: {pdf_path.name}\n\n')
                f.write(full_text)
            success += 1
        else:
            # 扫描版 PDF，无法提取文字
            with open(out_path, 'w', encoding='utf-8') as f:
                f.write(f'# 源文件: {pdf_path.name}\n# [扫描版PDF，无法提取文字]\n')
            failed += 1
    except Exception as e:
        failed += 1
        print(f'  ERROR [{i+1}/{total}] {pdf_path.name}: {e}', file=sys.stderr)
    
    if (i+1) % 100 == 0:
        print(f'  进度: {i+1}/{total} (成功:{success} 跳过:{skipped} 失败:{failed})')

print(f'完成! 成功:{success} 跳过:{skipped} 失败:{failed} 总计:{total}')