#!/usr/bin/env python3 """Audit semantic coverage between shipped features and agent skills.""" from __future__ import annotations import argparse import json from pathlib import Path from typing import Any SEVERITIES = ("critical", "high", "medium", "low") def _read_json(path: Path) -> Any: return json.loads(path.read_text(encoding="utf-8-sig")) def audit_feature_skills(repo_root: str | Path = ".") -> dict[str, Any]: root = Path(repo_root).resolve() contract_path = root / "templates" / "feature-skill-map.json" findings: list[dict[str, str]] = [] def add(severity: str, feature: str, path: str, message: str) -> None: findings.append({ "severity": severity if severity in SEVERITIES else "high", "feature": feature, "path": path, "message": message, }) try: contract = _read_json(contract_path) except (OSError, ValueError) as exc: add("critical", "contract", "templates/feature-skill-map.json", str(exc)) contract = {"features": []} if contract.get("schema_version") != "1.0": add("critical", "contract", "templates/feature-skill-map.json", "schema_version must be 1.0") # Tool/reachability parity (REQ-115, ai-collab-v3 REQ-116): every feature # names the tool-surface capabilities that expose it to agents, or carries # an explicit exemption rationale. Features can no longer ship toolless by # omission. capability_ids: set[str] = set() catalog_path = None for candidate in (root / "templates" / "tool-surface.json", root / ".prd_plugin" / "tool-surface.json"): if candidate.is_file(): catalog_path = candidate break if catalog_path is None: add("high", "contract", "templates/tool-surface.json", "tool-surface catalog missing: tool parity cannot be audited") else: try: catalog = _read_json(catalog_path) capability_ids = {str(c.get("id")) for c in catalog.get("capabilities", []) if isinstance(c, dict) and c.get("id")} except (OSError, ValueError) as exc: add("high", "contract", str(catalog_path), f"tool-surface catalog unreadable: {exc}") claimed_capabilities: set[str] = set() features = contract.get("features") if not isinstance(features, list) or not features: add("critical", "contract", "templates/feature-skill-map.json", "features must be a non-empty list") features = [] seen: set[str] = set() for feature in features: if not isinstance(feature, dict): add("critical", "contract", "templates/feature-skill-map.json", "feature entries must be objects") continue feature_id = str(feature.get("id", "")).strip() severity = str(feature.get("severity", "high")) if not feature_id: add("critical", "contract", "templates/feature-skill-map.json", "feature id is required") continue if feature_id in seen: add("critical", feature_id, "templates/feature-skill-map.json", "duplicate feature id") continue seen.add(feature_id) tool_capabilities = feature.get("tool_capabilities") tool_exemption = feature.get("tool_exemption") if bool(tool_capabilities) == bool(tool_exemption): add("high", feature_id, "templates/feature-skill-map.json", "tool parity undeclared: a feature must name tool_capabilities " "(tool-surface capability ids exposing it to agents) XOR a " "tool_exemption rationale") elif tool_capabilities: if not isinstance(tool_capabilities, list): add("critical", feature_id, "templates/feature-skill-map.json", "tool_capabilities must be an array of capability ids") else: for cap in tool_capabilities: if capability_ids and cap not in capability_ids: add("high", feature_id, "templates/tool-surface.json", f"tool parity broken: unknown tool capability {cap}") else: claimed_capabilities.add(cap) requirements = feature.get("requirements", []) if not isinstance(requirements, list) or not requirements: add("high", feature_id, "templates/feature-skill-map.json", "feature has no skill requirements") continue for requirement in requirements: if not isinstance(requirement, dict): add("critical", feature_id, "templates/feature-skill-map.json", "requirement entries must be objects") continue relative = str(requirement.get("path", "")) target = root / relative if not relative or not target.is_file(): add(severity, feature_id, relative or "", "required skill surface is missing") continue try: text = target.read_text(encoding="utf-8-sig") except (OSError, UnicodeError) as exc: add(severity, feature_id, relative, f"required skill surface could not be read: {exc}") continue contains = requirement.get("contains", []) not_contains = requirement.get("not_contains", []) if not isinstance(contains, list) or not isinstance(not_contains, list): add("critical", feature_id, "templates/feature-skill-map.json", "contains and not_contains must be arrays") continue for token in contains: if not isinstance(token, str): add("critical", feature_id, "templates/feature-skill-map.json", "guidance tokens must be strings") continue if token not in text: add(severity, feature_id, relative, f"missing required guidance: {token}") for token in not_contains: if not isinstance(token, str): add("critical", feature_id, "templates/feature-skill-map.json", "guidance tokens must be strings") continue if token in text: add(severity, feature_id, relative, f"obsolete or forbidden guidance remains: {token}") for unclaimed in sorted(capability_ids - claimed_capabilities): add("medium", "contract", "templates/tool-surface.json", f"tool parity gap: no feature claims tool capability {unclaimed}") counts = {severity: sum(row["severity"] == severity for row in findings) for severity in SEVERITIES} return { "status": "ok" if not findings else "findings", "repo_root": str(root), "contract": "templates/feature-skill-map.json", "feature_count": len(features), "summary": counts, "findings": findings, } def _markdown(report: dict[str, Any]) -> str: lines = [ "# Feature-to-Skill Audit", "", f"Status: `{report['status']}`", "", " ".join(f"{key.upper()}: {value}" for key, value in report["summary"].items()), ] for finding in report["findings"]: lines.append( f"- `{finding['severity']}` `{finding['feature']}` " f"[{finding['path']}]: {finding['message']}" ) return "\n".join(lines) + "\n" def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repo-root", default=".") parser.add_argument("--format", choices=("json", "markdown"), default="json") args = parser.parse_args(argv) report = audit_feature_skills(args.repo_root) print(json.dumps(report, indent=2) if args.format == "json" else _markdown(report), end="\n" if args.format == "json" else "") return 0 if report["status"] == "ok" else 1 if __name__ == "__main__": raise SystemExit(main())