"""Tests of the ingest pre-pass on PDFs built here, with PyMuPDF. No licensed norm can sit in the repository, so each test writes the PDF it needs: a native SIA layout (bookmarks, uppercase chapters, X.Y and X.Y.Z headings, numbered clauses, a running header, page numbers, a word hyphenated at a line end), a CEN layout (no bookmarks, sentence-case chapters, the number on its own line above the title), and a table whose rows look like headings. Run: python -m pytest packages/mcp/scripts/tests """ from __future__ import annotations import importlib.util import sys from pathlib import Path try: import pymupdf except ImportError: # PyMuPDF < 1.24.3 only knows the legacy name import fitz as pymupdf import pytest HERE = Path(__file__).resolve().parent SCRIPT = HERE.parent / "ingest-prepass.py" spec = importlib.util.spec_from_file_location("prepass", SCRIPT) prepass = importlib.util.module_from_spec(spec) sys.modules["prepass"] = prepass spec.loader.exec_module(prepass) # type: ignore[union-attr] def write_pdf(path: Path, pages: list[list[str]], toc: list[list] | None = None) -> None: """One text line per list entry, 16 pt apart, top to bottom.""" doc = pymupdf.open() for lines in pages: page = doc.new_page(width=595, height=842) y = 60 for line in lines: page.insert_text((56, y), line, fontsize=10, fontname="helv") y += 16 if toc: doc.set_toc(toc) doc.save(str(path)) doc.close() def run(pdf: Path, out: Path) -> dict: """The pre-pass as `main()` runs it, without the CLI.""" doc = pymupdf.open(str(pdf)) toc_pages = prepass.detect_toc_pages(doc) running = prepass.detect_running_text(doc) chapters = prepass.extract_chapters(doc, toc_pages) prefixes = {k for k in chapters if k.isdigit()} spans = prepass.chapter_page_spans(chapters, doc.page_count) subs = prepass.extract_subsections(doc, prefixes, toc_pages, running, spans) subs, pruned = prepass.prune_subsections(subs, chapters) nodes = prepass.build_tree(chapters, subs, doc.page_count) warnings = [f"pruned: {p}" for p in pruned] warnings += prepass.extract_section_text(doc, nodes, running) return {"nodes": nodes, "warnings": warnings, "chapters": chapters} HEADER = "SIA 267, Copyright 2013 by SIA Zurich" def sia_like(tmp_path: Path) -> Path: pdf = tmp_path / "sia.pdf" body = "Le terrain de fondation est caracterise par des essais." pages = [ [HEADER, "1 DOMAINE D'APPLICATION", "1.1 Objet", "1.1.1 La presente norme traite des fondations.", "1.1.2 Elle s'applique aux ouvrages neufs.", "3"], [HEADER, "2 TERMINOLOGIE", "2.1 Termes techniques", "2.1.1 Pieu : element de fondation elance.", "4"], [HEADER, "9 FONDATIONS SUR PIEUX", "9.1 Delimitation", "9.1.1 Ce chapitre traite des pieux fores et battus.", "9.5 Dimensionnement", "9.5.1 Generalites", "9.5.1.1 " + body, "5"], [HEADER, "9.5.2 Resistance ultime du pieu isole", "9.5.2.1 La resistance ultime se compose de la resistance de pointe et du frot-", "tement lateral, voir chiffre 9.5.3.", "9.5.3 Frottement lateral", "9.5.3.1 Le frottement lateral se determine par essais.", "6"], [HEADER, "9.6 Dispositions d'execution", "9.6.1 Les pieux sont executes selon la norme SIA 267/1.", "7"], [HEADER, "Annexe A (normative)", "A.1 Valeurs indicatives", "Le tableau donne des valeurs indicatives.", "8"], ] toc = [ [1, "1 DOMAINE D'APPLICATION", 1], [1, "2 TERMINOLOGIE", 2], [1, "9 FONDATIONS SUR PIEUX", 3], [1, "Annexe A (normative)", 6], ] write_pdf(pdf, pages, toc) return pdf def test_tree_descends_to_titled_clauses(tmp_path): result = run(sia_like(tmp_path), tmp_path) paths = [n["path"] for n in result["nodes"]] assert paths[:3] == ["1", "1.1", "2"] assert "9.5" in paths and "9.5.1" in paths and "9.5.2" in paths and "9.5.3" in paths assert "9.6" in paths and "Annexe A" in paths # Numbered paragraphs are clauses inside their heading, not nodes. assert "9.5.2.1" not in paths and "1.1.1" not in paths by_path = {n["path"]: n for n in result["nodes"]} assert by_path["9.5.2"]["depth"] == 2 assert by_path["9.5.2"]["parentNodeId"] == by_path["9.5"]["nodeId"] def test_each_node_carries_only_its_own_text(tmp_path): result = run(sia_like(tmp_path), tmp_path) by_path = {n["path"]: n["rawText"] for n in result["nodes"]} # The chapter holds its intro, not the text of every section under it. assert "9.5.2.1" not in by_path["9"] assert "Delimitation" not in by_path["9"] or by_path["9"].count("\n") < 3 # A section holds its clauses up to the next heading, whatever its depth. assert "9.5.2.1 La resistance ultime" in by_path["9.5.2"] assert "9.5.3.1" not in by_path["9.5.2"] assert "9.5.3.1 Le frottement lateral" in by_path["9.5.3"] # 9.5.1 gets its clause, 9.5 only what is between its heading and 9.5.1. assert "9.5.1.1" in by_path["9.5.1"] assert "9.5.1.1" not in by_path["9.5"] def test_pages_are_cleaned(tmp_path): result = run(sia_like(tmp_path), tmp_path) for node in result["nodes"]: assert HEADER not in node["rawText"], node["path"] # Bare page numbers are gone; the clause numbers are joined to their text. assert not any(line.strip().isdigit() for line in node["rawText"].splitlines()), node["path"] text = {n["path"]: n["rawText"] for n in result["nodes"]}["9.5.2"] # Hyphenation at the line end is resolved by PyMuPDF. assert "frottement lateral" in text assert "frot-" not in text assert result["warnings"] == [] def cen_like(tmp_path: Path) -> Path: """No bookmarks; the number sits on its own line above a sentence-case title.""" pdf = tmp_path / "cen.pdf" pages = [ ["1", "Domaine d'application", "Le present document specifie les exigences.", "2", "References normatives", "EN 1997-1, Calcul geotechnique."], ["3", "Termes et definitions", "3.1", "Ancrage", "Element transmettant une force de traction au terrain.", "3.2", "Tirant", "Partie libre de l'ancrage."], ["4", "Execution", "4.1", "Forage", "Le forage est realise sans deblaiement."], ] write_pdf(pdf, pages) return pdf def test_cen_layout_without_bookmarks(tmp_path): result = run(cen_like(tmp_path), tmp_path) paths = [n["path"] for n in result["nodes"]] assert paths[:2] == ["1", "2"] assert "3" in paths and "3.1" in paths and "3.2" in paths and "4.1" in paths by_path = {n["path"]: n["rawText"] for n in result["nodes"]} assert "Element transmettant" in by_path["3.1"] assert "Partie libre" not in by_path["3.1"] def table_like(tmp_path: Path) -> Path: """A chapter whose table rows look like numbered headings.""" pdf = tmp_path / "table.pdf" pages = [ ["1 GENERALITES", "1.1 Objet", "La presente norme fixe les prestations.", "2 PRESTATIONS", "2.1 Etudes preliminaires", "Tableau 1 : phases", "2.3 retrait des obstacles", "2.2 Avant-projet", "Description de la phase."], ["3 HONORAIRES", "3.1 Calcul", "Le calcul se fait au temps employe."], ] write_pdf(pdf, pages) return pdf def test_table_rows_are_not_headings(tmp_path): result = run(table_like(tmp_path), tmp_path) paths = [n["path"] for n in result["nodes"]] # A lowercase numbered line is a table row. assert "2.3" not in paths assert "2.1" in paths and "2.2" in paths and "3.1" in paths def test_prune_subsections_drops_out_of_sequence_child(): chapters = {"4": {"title": "X", "pageStart": 10}, "9": {"title": "Y", "pageStart": 60}} subs = { "4.1": ("Premiere", 11), "4.2": ("Deuxieme", 63), # found inside chapter 9: a table row "4.3": ("Troisieme", 14), "9.1": ("Une", 61), } kept, dropped = prepass.prune_subsections(subs, chapters) assert set(kept) == {"4.1", "4.3", "9.1"} assert len(dropped) == 1 and dropped[0].startswith("4.2") def test_join_clause_numbers_and_page_cleaning(): joined = prepass.join_clause_numbers("9.5.1.1\nLes pieux sont fores.\n9.5.1.2\nIls sont battus.") assert joined == "9.5.1.1 Les pieux sont fores.\n9.5.1.2 Ils sont battus." cleaned = prepass.clean_page("SIA 267, Copyright\nTexte\n63", {"SIA 267, Copyright"}, 110) assert cleaned == "Texte" # A number in the middle of a long page is content, not a page number. middle = "\n".join(["a", "b", "c", "d", "42", "e", "f", "g", "h"]) assert "42" in prepass.clean_page(middle, set(), 110) def test_recovers_chapter_numbers_by_position(tmp_path): """An OCR'd copy loses the numbering column on a run of pages: the chapter titles survive without their number, and so do their sub-headings.""" pdf = tmp_path / "ocr.pdf" body = "Le terrain de fondation est caracterise par des essais." pages = [ ["DOMAINE D'APPLICATION", "Delimitation", "La norme traite des fondations.", "1"], ["1 TERMINOLOGIE", "1.1 Termes techniques", "Pieu : element de fondation elance.", "2"], ["2 PRINCIPES", "2.1 Generalites", body, "3"], ["TERRAIN DE FONDATION", "Generalites", body, "4"], ["ANALYSE STRUCTURALE", "Generalites", body, "5"], ["5 DIMENSIONNEMENT", "5.1 Generalites", body, "6"], ] write_pdf(pdf, pages) chapters = run(pdf, tmp_path)["chapters"] assert chapters["3"]["title"] == "TERRAIN DE FONDATION" assert chapters["3"]["pageStart"] == 4 assert chapters["4"]["title"] == "ANALYSE STRUCTURALE" assert chapters["4"]["pageStart"] == 5 assert chapters["0"]["pageStart"] == 1 assert prepass.missing_chapter_numbers(chapters) == [] WATERMARK = "Ecole Polytechnique Federale de Lausanne, EPFL, SNV / licensed to" def test_licence_marks_are_kept_once(tmp_path): """The footer and the vertical watermark leave the text, never the norm.""" pdf = sia_like(tmp_path) doc = pymupdf.open(str(pdf)) for page in doc: page.insert_text((20, 700), WATERMARK, fontsize=7, fontname="helv", rotate=90) marked = tmp_path / "marked.pdf" doc.save(str(marked)) doc.close() doc = pymupdf.open(str(marked)) licence = prepass.detect_licence(doc, prepass.detect_running_text(doc)) lines = licence.splitlines() assert HEADER in lines assert WATERMARK in lines assert len(lines) == len(set(lines))