#!/usr/bin/env python3 """ pdf-text-extract.py — PDF text extraction for SpecMem codebase indexing Uses PyMuPDF (fitz) for instant digital PDF text extraction (0.003s/page). Falls back to Tesseract OCR via PyMuPDF's built-in integration for scanned pages. Usage: # Single file mode python3 pdf-text-extract.py [--max-pages N] [--language LANG] # Batch mode (JSONL — one result per line, one Python startup for N PDFs) python3 pdf-text-extract.py --batch file1.pdf file2.pdf ... [--max-pages N] Output (JSON/JSONL to stdout): {"path": "/abs/path.pdf", "text": "...", "pages": 5, "scanned_pages": [3], "chars": 12345} {"path": "/abs/path2.pdf", "error": "..."} """ import sys import json import os import argparse # --------------------------------------------------------------------------- # Auto-install pymupdf if missing (matches frankenstein-embeddings pattern) # --------------------------------------------------------------------------- def _ensure_pymupdf(): try: import pymupdf return pymupdf except ImportError: pass # Try legacy import name try: import fitz return fitz except ImportError: pass # Auto-install try: import subprocess sys.stderr.write('[pdf-text-extract] pymupdf not found, installing...\n') subprocess.check_call( [sys.executable, '-m', 'pip', 'install', '--quiet', 'pymupdf'], stdout=subprocess.DEVNULL ) try: import pymupdf return pymupdf except ImportError: import fitz return fitz except Exception as e: _error_exit(f'Failed to install pymupdf: {e}') def _error_exit(msg): """Print error JSON and exit.""" print(json.dumps({'error': str(msg)}, ensure_ascii=False)) sys.exit(1) def _is_scanned_page(page, text): """ Heuristic: page is likely scanned if: 1. Extracted text is very short (< 50 chars after stripping) 2. Page has images covering >60% of page area """ stripped = text.strip() if len(stripped) > 50: return False try: images = page.get_image_info() if not images: return False page_area = abs(page.rect) if page_area == 0: return False image_area = 0 for img in images: if 'bbox' in img: try: import pymupdf r = pymupdf.Rect(img['bbox']) except (ImportError, Exception): import fitz r = fitz.Rect(img['bbox']) image_area += abs(r) return (image_area / page_area) >= 0.6 except Exception: return False def _ocr_page(page, language='eng'): """ Attempt Tesseract OCR on a scanned page via PyMuPDF's built-in integration. Returns extracted text or empty string if tesseract unavailable. """ try: tp = page.get_textpage_ocr(language=language, dpi=300) return page.get_text(textpage=tp).strip() except Exception as e: msg = str(e).lower() if 'tesseract' in msg or 'not installed' in msg or 'not found' in msg: # Tesseract not installed — skip OCR, return what we have sys.stderr.write(f'[pdf-text-extract] Tesseract not available, skipping OCR for scanned page\n') return '' # Other error — still don't crash sys.stderr.write(f'[pdf-text-extract] OCR failed: {e}\n') return '' def extract_pdf(pdf_path, max_pages=100, language='eng'): """ Extract text from PDF using PyMuPDF. Digital pages: instant text extraction. Scanned pages: Tesseract OCR fallback. TODO: Add support for extracting metadata (author, title, creation date) from PDF """ pymupdf = _ensure_pymupdf() if not os.path.isfile(pdf_path): return {'error': f'File not found: {pdf_path}'} try: doc = pymupdf.open(pdf_path) except Exception as e: msg = str(e).lower() if 'password' in msg or 'encrypt' in msg: return {'error': f'PDF is password-protected: {pdf_path}'} return {'error': f'Failed to open PDF: {e}'} total_pages = len(doc) process_count = min(total_pages, max_pages) truncated = total_pages > max_pages texts = [] scanned_pages = [] for i in range(process_count): page = doc[i] text = page.get_text().strip() if _is_scanned_page(page, text): # Try OCR ocr_text = _ocr_page(page, language) if ocr_text: text = ocr_text scanned_pages.append(i + 1) # 1-indexed # If OCR also empty, keep whatever minimal text we got if text: if process_count > 1: texts.append(f'--- Page {i + 1} ---\n{text}') else: texts.append(text) doc.close() full_text = '\n\n'.join(texts) result = { 'text': full_text, 'pages': process_count, 'chars': len(full_text), } if scanned_pages: result['scanned_pages'] = scanned_pages if truncated: result['truncated'] = True result['total_pages'] = total_pages return result def main(): parser = argparse.ArgumentParser(description='Extract text from PDF files') parser.add_argument('pdf_path', nargs='?', help='Path to the PDF file (single mode)') parser.add_argument('--batch', nargs='+', metavar='PDF', help='Batch mode: extract multiple PDFs (JSONL output, one line per PDF)') parser.add_argument('--max-pages', type=int, default=100, help='Maximum pages to process per PDF (default: 100)') parser.add_argument('--language', default='eng', help='Tesseract language for OCR fallback (default: eng)') args = parser.parse_args() if args.batch: # Batch mode — JSONL output, one result per line # Single Python startup for N PDFs (avoids repeated interpreter overhead) for pdf_path in args.batch: result = extract_pdf(pdf_path, args.max_pages, args.language) result['path'] = pdf_path print(json.dumps(result, ensure_ascii=False), flush=True) elif args.pdf_path: # Single file mode result = extract_pdf(args.pdf_path, args.max_pages, args.language) result['path'] = args.pdf_path print(json.dumps(result, ensure_ascii=False)) else: parser.print_help() sys.exit(1) if __name__ == '__main__': main()