#!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # Copyright 2024-2026 Lukasz Krzemien (biuro@softspark.eu) # Source: https://github.com/softspark/ai-toolkit """Split gate for skill refactors. Proves that moving content out of a SKILL.md body into reference/ lost nothing. Run it after every body -> reference/ split, before the change is considered done. Four gates: A code preservation Every fenced code-block line from the pre-split body still exists in the body + reference/ union. Commands, snippets and templates are the risk surface — losing one silently changes what the skill tells the model to run. B content trace Every non-blank line removed from the body reappears in reference/. Warn by default (prose gets reworded into pointers during a legitimate split), error under --strict. C always-loaded Sections that must load on every run stayed in the body: sections Rules, Gotchas, When NOT to Use. Only sections that existed before the split are required after it. D routing stability Frontmatter description is byte-identical. The description decides which prompts select the skill, so changing it during a refactor is a behavioural change wearing a refactor's clothes. E link integrity Every relative link in the body and in reference/ resolves from the file that holds it. A split invalidates every `(reference/x.md)` path that moves down a directory, and validate.py only checks links in SKILL.md — so a broken pointer inside reference/ ships silently. Stdlib-only. Human-readable text to stdout; --json for machine use. Usage: python3 scripts/check_split.py --before python3 scripts/check_split.py --base-ref HEAD python3 scripts/check_split.py --base-ref HEAD --strict python3 scripts/check_split.py --base-ref HEAD --json python3 scripts/check_split.py --base-ref HEAD --toolkit-dir /path Exit codes: 0 all gates passed 1 a gate failed 2 usage error (unknown skill, missing/duplicate source, unreadable git ref) """ from __future__ import annotations import json import re import subprocess import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from _common import toolkit_dir as default_toolkit_dir # Sections that carry prescriptive process and environment-specific traps. # They must load on every run, so they never move down into reference/. ALWAYS_LOADED_SECTIONS = ("Rules", "Gotchas", "When NOT to Use") HEADING_RE = re.compile(r"^\s{0,3}#{1,6}\s+(.*?)\s*#*\s*$") FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})") LINK_RE = re.compile(r"\]\(([^)\s]+)\)") # Inline code spans are literal text. A regex such as # `type=["']password["'](?!.*autocomplete)` reads as a link to the naive matcher. INLINE_CODE_RE = re.compile(r"`+[^`]*`+") # Link targets that are not files on disk and must not be resolved. NON_FILE_PREFIXES = ("#", "/", "mailto:", "tel:", "$") # --------------------------------------------------------------------------- # Parsing helpers # --------------------------------------------------------------------------- def split_frontmatter(text: str) -> tuple[str, str]: """Return (frontmatter_block, body). Frontmatter is '' when absent.""" lines = text.splitlines() if not lines or lines[0].strip() != "---": return "", text for idx in range(1, len(lines)): if lines[idx].strip() == "---": return "\n".join(lines[1:idx]), "\n".join(lines[idx + 1:]) # Unterminated frontmatter — treat the whole file as body rather than # silently swallowing it. return "", text def description_of(frontmatter: str) -> str: """Extract the `description:` value, unquoted, from a frontmatter block.""" for line in frontmatter.splitlines(): if not line.startswith("description:"): continue value = line[len("description:"):].strip() if len(value) >= 2 and value[0] == value[-1] and value[0] in ("\"", "'"): value = value[1:-1] return value return "" def fenced_code_lines(body: str) -> list[str]: """Return the lines strictly inside fenced code blocks, stripped, non-empty. Nested fences are not a thing in CommonMark: the first fence opens, a fence of the same character closes. Tracking the opening character avoids treating a ``` inside a ~~~ block as a terminator. """ out: list[str] = [] fence_char = "" for line in body.splitlines(): match = FENCE_RE.match(line) if match: marker = match.group(1)[0] if not fence_char: fence_char = marker continue if marker == fence_char: fence_char = "" continue if fence_char: stripped = line.strip() if stripped: out.append(stripped) return out def normalize(line: str) -> str: """Normalize a prose line for comparison. Headings lose their leading hashes so that a section promoted from `###` in the body to `##` in a reference file still matches. Internal whitespace runs collapse so that reflowed markdown does not read as lost content. """ stripped = line.strip() heading = HEADING_RE.match(stripped) if heading: stripped = heading.group(1).strip() return " ".join(stripped.split()) def nonblank_normalized(text: str) -> list[str]: return [n for n in (normalize(line) for line in text.splitlines()) if n] def section_headings(body: str) -> set[str]: """Return the set of heading titles present in a body.""" out: set[str] = set() fence_char = "" for line in body.splitlines(): match = FENCE_RE.match(line) if match: marker = match.group(1)[0] if not fence_char: fence_char = marker elif marker == fence_char: fence_char = "" continue if fence_char: continue heading = HEADING_RE.match(line) if heading: out.add(heading.group(1).strip()) return out # --------------------------------------------------------------------------- # Gates # --------------------------------------------------------------------------- def gate_a_code(before_body: str, union_lines: set[str]) -> dict: """Every fenced code line from the pre-split body survives somewhere.""" missing = [line for line in fenced_code_lines(before_body) if line not in union_lines] # Preserve order, drop repeats — a line repeated in three snippets is one # loss to fix, not three. seen: set[str] = set() unique = [line for line in missing if not (line in seen or seen.add(line))] return { "status": "pass" if not unique else "fail", "code_lines_lost": len(unique), "missing": unique, } def gate_b_trace(before_body: str, after_body: str, reference_text: str, strict: bool) -> dict: """Every non-blank line removed from the body reappears in reference/.""" after_set = set(nonblank_normalized(after_body)) reference_set = set(nonblank_normalized(reference_text)) reference_blob = "\n".join(reference_set) unmatched: list[str] = [] seen: set[str] = set() for line in nonblank_normalized(before_body): if line in after_set or line in seen: continue seen.add(line) if line in reference_set: continue # Substring match catches a pointer that reworded the original line # while keeping its content, e.g. a heading folded into a sentence. if line in reference_blob: continue unmatched.append(line) if not unmatched: status = "pass" else: status = "fail" if strict else "warn" return { "status": status, "unmatched_count": len(unmatched), "unmatched": unmatched, } def gate_c_sections(before_body: str, after_body: str) -> dict: """Sections that must load on every run did not move into reference/.""" before_headings = section_headings(before_body) after_headings = section_headings(after_body) missing = [ name for name in ALWAYS_LOADED_SECTIONS if name in before_headings and name not in after_headings ] return { "status": "pass" if not missing else "fail", "missing": missing, "checked": [name for name in ALWAYS_LOADED_SECTIONS if name in before_headings], } def markdown_links(text: str) -> list[str]: """Markdown link targets that sit outside fenced code blocks. Links inside a fence are examples, not pointers — resolving them would flag every sample path a skill documents. """ out: list[str] = [] fence = "" for line in text.splitlines(): match = FENCE_RE.match(line) if match: marker = match.group(1)[0] fence = "" if fence == marker else (fence or marker) continue if fence: continue out.extend(LINK_RE.findall(INLINE_CODE_RE.sub(" ", line))) return out def gate_e_links(skill_dir: Path, after_body: str) -> dict: """Every relative link resolves from the file that holds it. Checked across the body and every reference/*.md, because moving a section down one directory silently invalidates the paths it carried, and validate.py's own link check never looks inside reference/. """ sources: list[tuple[Path, str]] = [(skill_dir / "SKILL.md", after_body)] ref_dir = skill_dir / "reference" if ref_dir.is_dir(): for path in sorted(ref_dir.rglob("*.md")): sources.append((path, path.read_text(encoding="utf-8"))) broken: list[str] = [] checked = 0 for path, text in sources: for target in markdown_links(text): if "://" in target or target.startswith(NON_FILE_PREFIXES): continue relative = target.split("#", 1)[0] if not relative: continue checked += 1 if not (path.parent / relative).exists(): broken.append(f"{path.name} -> {target}") return { "status": "pass" if not broken else "fail", "links_checked": checked, "broken_count": len(broken), "broken": broken, } def gate_d_description(before_fm: str, after_fm: str) -> dict: """Routing must not move during a refactor.""" before = description_of(before_fm) after = description_of(after_fm) return { "status": "pass" if before == after else "fail", "before": before, "after": after, } # --------------------------------------------------------------------------- # Sources # --------------------------------------------------------------------------- def read_before_from_git(tk_dir: Path, skill: str, ref: str) -> str: """Read the pre-split SKILL.md out of a git ref.""" rel = f"app/skills/{skill}/SKILL.md" try: result = subprocess.run( ["git", "-C", str(tk_dir), "show", f"{ref}:{rel}"], capture_output=True, text=True, check=False, ) except (OSError, ValueError) as exc: # git missing, bad argument shape fail_usage(f"cannot run git: {exc}") if result.returncode != 0: fail_usage( f"cannot read {rel} at ref '{ref}': " f"{result.stderr.strip() or 'git exited ' + str(result.returncode)}" ) return result.stdout def read_reference_text(skill_dir: Path) -> tuple[str, list[str]]: """Concatenate every reference/*.md file. Returns (text, filenames).""" ref_dir = skill_dir / "reference" if not ref_dir.is_dir(): return "", [] chunks: list[str] = [] names: list[str] = [] for path in sorted(ref_dir.rglob("*.md")): chunks.append(path.read_text(encoding="utf-8")) names.append(str(path.relative_to(skill_dir))) return "\n".join(chunks), names # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- def fail_usage(message: str) -> None: print(f"ERROR: {message}", file=sys.stderr) sys.exit(2) def parse_args(argv: list[str]) -> dict: opts = { "skill": "", "before": "", "base_ref": "", "toolkit_dir": str(default_toolkit_dir), "json": False, "strict": False, } idx = 0 while idx < len(argv): arg = argv[idx] if arg == "--json": opts["json"] = True elif arg == "--strict": opts["strict"] = True elif arg in ("-h", "--help"): print(__doc__) sys.exit(0) elif arg in ("--before", "--base-ref", "--toolkit-dir"): idx += 1 if idx >= len(argv): fail_usage(f"{arg} requires a value") key = arg.lstrip("-").replace("-", "_") opts[key] = argv[idx] elif arg.startswith("-"): fail_usage(f"unknown option: {arg}") elif not opts["skill"]: opts["skill"] = arg else: fail_usage(f"unexpected argument: {arg}") idx += 1 return opts def report_text(result: dict) -> None: gates = result["gates"] print(f"## Split gate — {result['skill']}") print() code = gates["A_code_preservation"] print(f" code lines lost: {code['code_lines_lost']}") for line in code["missing"][:20]: print(f" LOST: {line}") if len(code["missing"]) > 20: print(f" ... and {len(code['missing']) - 20} more") trace = gates["B_content_trace"] label = "ERROR" if trace["status"] == "fail" else "WARN" if trace["unmatched_count"]: print(f" {label}: {trace['unmatched_count']} removed lines not found in reference/") for line in trace["unmatched"][:20]: print(f" UNTRACED: {line}") if trace["unmatched_count"] > 20: print(f" ... and {trace['unmatched_count'] - 20} more") else: print(" OK: every removed line traced to reference/") sections = gates["C_always_loaded_sections"] if sections["missing"]: print(f" ERROR: always-loaded sections left the body: {', '.join(sections['missing'])}") else: checked = ", ".join(sections["checked"]) or "none present" print(f" OK: always-loaded sections intact ({checked})") desc = gates["D_description_stable"] if desc["status"] == "fail": print(" ERROR: frontmatter description changed — routing moved") print(f" before: {desc['before']}") print(f" after: {desc['after']}") else: print(" OK: description unchanged") links = gates["E_link_integrity"] if links["broken"]: print(f" ERROR: {links['broken_count']} relative links do not resolve") for entry in links["broken"][:20]: print(f" BROKEN: {entry}") if links["broken_count"] > 20: print(f" ... and {links['broken_count'] - 20} more") else: print(f" OK: {links['links_checked']} relative links resolve") print() print(f" body: {result['body_before_bytes']} -> {result['body_after_bytes']} bytes") print(f" reference files: {len(result['reference_files'])}") print() print("SPLIT GATE PASSED" if result["ok"] else "SPLIT GATE FAILED") def main(argv: list[str]) -> int: opts = parse_args(argv) if not opts["skill"]: fail_usage("missing skill name") if bool(opts["before"]) == bool(opts["base_ref"]): fail_usage("pass exactly one of --before or --base-ref ") tk_dir = Path(opts["toolkit_dir"]).resolve() skill_dir = tk_dir / "app" / "skills" / opts["skill"] skill_file = skill_dir / "SKILL.md" if not skill_file.is_file(): fail_usage(f"no SKILL.md at {skill_file}") if opts["before"]: before_path = Path(opts["before"]) if not before_path.is_file(): fail_usage(f"--before file not found: {before_path}") before_text = before_path.read_text(encoding="utf-8") else: before_text = read_before_from_git(tk_dir, opts["skill"], opts["base_ref"]) after_text = skill_file.read_text(encoding="utf-8") before_fm, before_body = split_frontmatter(before_text) after_fm, after_body = split_frontmatter(after_text) reference_text, reference_files = read_reference_text(skill_dir) union_lines = { line.strip() for line in (after_body + "\n" + reference_text).splitlines() if line.strip() } gates = { "A_code_preservation": gate_a_code(before_body, union_lines), "B_content_trace": gate_b_trace(before_body, after_body, reference_text, opts["strict"]), "C_always_loaded_sections": gate_c_sections(before_body, after_body), "D_description_stable": gate_d_description(before_fm, after_fm), "E_link_integrity": gate_e_links(skill_dir, after_body), } ok = all(gate["status"] != "fail" for gate in gates.values()) result = { "skill": opts["skill"], "strict": opts["strict"], "gates": gates, "body_before_bytes": len(before_body.encode("utf-8")), "body_after_bytes": len(after_body.encode("utf-8")), "reference_files": reference_files, "ok": ok, } if opts["json"]: print(json.dumps(result, indent=2, ensure_ascii=False)) else: report_text(result) return 0 if ok else 1 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))