/** * `dz score --slug ` — score a feature-adr RUN's process discipline (feature dz-score, * Reading C of `features/dz-score/PROPOSAL.md`, chosen by the user 2026-07-28). * * It scores the PROCESS, not the code: were the ADR's safety properties given a named test? was * discrimination proven? did cross-model QE happen and what did it say? was the work verified live? * did the READMEs travel in the same change? did the learning loop run? * * Readings A (repo dashboard) and B (skill scoring) were rejected in the proposal: A invites * Goodharting the gates, B would be a fourth scoring surface. C is hard to game — the only way to * score well is to actually run the discipline. * * DESCRIPTIVE-ONLY, permanently: the command never gates, never exits non-zero on a low score. * The health-advisor 1.2.0 run is the reference case: its QE report marked the registration * criterion "✅ (mechanism)" with no live evidence — this scorecard exists to make that visible. * * Discriminators were chosen from a SURVEY of the 132 real runs on disk (34/77 ADRs carry a * Confirmation section; 31/75 QE reports carry MEASURED markers) — not guessed. * * PURE: the CLI reads the artifact files; this module only classifies. */ export type DisciplineVerdict = 'pass' | 'partial' | 'absent'; export interface DisciplineScore { readonly id: string; readonly title: string; readonly verdict: DisciplineVerdict; /** The line of evidence the verdict rests on — a scorecard must show its work. */ readonly evidence: string; } export interface RunScorecard { readonly slug: string; readonly disciplines: readonly DisciplineScore[]; /** Extracted cross-model grade, when one exists (e.g. "A−", "C"). */ readonly qeGrade: string | null; readonly passed: number; readonly total: number; readonly summary: string; } /** The artifact texts of one run, keyed by RELATIVE path under `features//`. */ export type RunArtifacts = Readonly>; function collect(artifacts: RunArtifacts, predicate: (path: string) => boolean): string { return Object.entries(artifacts) .filter(([p]) => predicate(p)) .map(([, text]) => text) .join('\n'); } /** First matching line (trimmed, capped) — the evidence a verdict shows. */ function evidenceLine(text: string, re: RegExp): string | null { for (const line of text.split('\n')) { if (re.test(line)) return line.trim().slice(0, 140); } return null; } /** * Like {@link evidenceLine}, but a NEGATED mention is not evidence: "Codex was not used" and * "no discrimination proof was performed" both satisfied the plain regexes (Codex QE #1, and its * heuristic table). A line whose match is preceded by a negation word is skipped. Heuristic — but * the failure mode flips from a silent false pass to a visible miss the shown evidence exposes. */ // The DEFAULT vocabulary — verbs and determiners that deny the sentence they sit in. const NEGATION_RE = /\b(no|not|never|without|wasn'?t|isn'?t)\b/i; /** * The default vocabulary plus the negative QUANTIFIERS. Opt-in per site, because a quantifier * negates a NOUN, not the claim: "None of the mutants survived; discrimination §42 is proven by the * red run" is idiomatic POSITIVE evidence that the wide list silently discarded (QE B-F2 — * negating-the-mutants is not negating-the-proof). It is passed only where a red test demanded it: * "Nothing was MEASURED in this round" scored as proof of measurement, because the word boundary in * `\bno\b` does NOT match "Nothing". Hedges like "skipped" stay out of both lists — they routinely * appear inside genuine evidence lines. */ // RU negation quantifiers joined 2026-08-24 with the RU live-markers (773185ca): a corpus where // 63% of traffic is Russian was screened by an English-only list — «ничего не измерено» would have // read as a live marker the moment ИЗМЕРЕНО joined the positives. // \b is ASCII-only in JS even under /u — «не» never matched through it (measured by the pin the // moment it was written). Unicode lookarounds carry the boundary instead. const NEGATION_QUANTIFIED_RE = /\b(no|not|never|without|nothing|none|neither|nor|nobody|wasn'?t|isn'?t)\b|(? { let lo = 0, hi = lineStarts.length - 1; while (lo < hi) { const mid = (lo + hi + 1) >> 1; if ((lineStarts[mid] as number) <= idx) lo = mid; else hi = mid - 1; } return qeText.slice(lineStarts[lo] as number, idx); }; for (const m of qeText.matchAll(new RegExp(GRADE_RE.source, 'g'))) { if (NEGATION_QUANTIFIED_RE.test(linePrefixOf(m.index ?? 0))) continue; const g = normaliseGradeSign(m[1] as string); if (!found.includes(g)) found.push(g); } if (found.length === 0) return { status: 'none', grade: null, found: [] }; if (found.length === 1) return { status: 'unique', grade: found[0] as string, found }; return { status: 'ambiguous', grade: null, found }; } export function extractQeGrade(qeText: string): string | null { return readQeGrade(qeText).grade; } export function scoreRun(slug: string, artifacts: RunArtifacts): RunScorecard { const adrText = collect(artifacts, (p) => p.startsWith('03_adr/')); const qeText = collect(artifacts, (p) => p === '08_qe_report.md' || p === '09_fleet_qe_assessment.md'); const planText = collect(artifacts, (p) => p === '06_implementation_plan.md' || p === '03.5_ideation_report.md'); const complexityText = collect(artifacts, (p) => p === '00_complexity_assessment.md'); const manifestText = collect(artifacts, (p) => p.startsWith('07_code_changes/')); const allText = collect(artifacts, () => true); const disciplines: DisciplineScore[] = []; const add = (id: string, title: string, verdict: DisciplineVerdict, evidence: string): void => { disciplines.push({ id, title, verdict, evidence }); }; // 1. ADR with a Confirmation — a named decision whose load-bearing property names its test. if (adrText === '') { add('adr-confirmation', 'ADR present, property → named test', 'absent', 'no 03_adr/*.md artifact'); } else { // POSITIVE (QE B-F2 reversed my first call, which exempted this site as "structural"). The // heading regex allows a SUFFIX, so `## Confirmation — not yet performed` — the realistic // placeholder an unfinished ADR carries — scored a full PASS. Heading presence is structural; // heading TEXT is not, and this one can deny itself. Default (narrow) vocabulary: a heading is // a fragment, and the quantifiers only appear in prose. Pinned both ways by tests. const conf = evidenceLinePositive(adrText, /^##+\s*Confirmation/i); add( 'adr-confirmation', 'ADR present, property → named test', conf !== null ? 'pass' : 'partial', conf ?? 'ADR exists but has no (non-negated) Confirmation heading — the load-bearing property names no test', ); } // 2. Discrimination — proof the test can FAIL (the §42 gate, or an explicit mutation proof). // Default (narrow) vocabulary ON PURPOSE (QE B-F2): mutation evidence is written by negating the // MUTANTS — "None of the mutants survived", "neither mutant escaped" — which is the proof, not // its denial. The quantifiers would discard exactly the strongest lines this discipline exists // to find. "No discrimination proof was performed" is still caught by the narrow list. const discr = evidenceLinePositive(allText, /discrimination|§42/i) ?? evidenceLinePositive(allText, /mutation[s]?\s.*(prov|kill)|mutant[s]?\s.*(kill|red)|RED on the old|goes? RED|failed as expected/i); add( 'discrimination', 'the property test is proven able to fail', discr !== null ? 'pass' : 'absent', discr ?? 'no discrimination/§42/mutation-proof evidence in any artifact', ); // 3. Cross-model QE — an independent family reviewed it, and a grade exists. const grade = extractQeGrade(qeText); if (qeText === '') { add('cross-model-qe', 'independent cross-model review with a grade', 'absent', 'no 08_qe_report.md artifact'); } else { const crossLine = evidenceLinePositive(qeText, /codex|gpt-|cross-model/i); // EXEMPT from the negation filter (wave1-scorer-negation, per-site review): this line is a // DISPLAY LOCATOR, not a verdict input — the verdict above rests on `crossLine` (already // positive-filtered) AND on `grade`, parsed from the whole report. A letter grade is a // structural token ("Grade: D"); there is no idiomatic "no Grade: D". Filtering here would only // drop the most common real grade line ("**Grade: B** — no blockers remain") from the shown // evidence for zero change in verdict. Pinned by a test. Residual, flagged not hidden: // `extractQeGrade` itself is negation-blind and stays so — out of FR-B1's scope. const gradeLine = grade !== null ? evidenceLine(qeText, GRADE_RE) : null; add( 'cross-model-qe', 'independent cross-model review with a grade', crossLine !== null && grade !== null ? 'pass' : 'partial', crossLine !== null ? grade !== null ? `${crossLine}${gradeLine !== null && gradeLine !== crossLine ? ` | ${gradeLine}` : ''}`.slice(0, 140) : `${crossLine} — but NO parseable grade`.slice(0, 140) : 'QE report exists but no (non-negated) cross-model reviewer is named (self-review only)', ); } // 4. Live verification — the property was observed, not inferred. The health-advisor 1.2.0 QE // report is the cautionary case: "✅ (mechanism)" with no live evidence shipped a dead feature. // POSITIVE (wave1-scorer-negation): "nothing was MEASURED" / "no reproducer was run" is the // claim's exact opposite and used to score as proof of it (the crossrt-1 6/7 shape). // 773185ca: MEASURED-class markers in BOTH working languages. dz-recap carried 10× ИЗМЕРЕНО and // 0× MEASURED and scored «всё выведено рассуждением» — the cyrillic-tokenizer class again. const LIVE_MARKER_RE = /MEASURED|verified live|VERIFIED LIVE|reproducer|ИЗМЕРЕНО|МЕРЕНО|измерено|проверено живьём|живой прогон|репродьюсер/iu; const live = evidenceLinePositive(qeText, LIVE_MARKER_RE, NEGATION_QUANTIFIED_RE); const liveAnywhere = live ?? evidenceLinePositive(allText, LIVE_MARKER_RE, NEGATION_QUANTIFIED_RE); add( 'live-verification', 'claims verified by running, not by reasoning', live !== null ? 'pass' : liveAnywhere !== null ? 'partial' : 'absent', live ?? liveAnywhere ?? 'no MEASURED/ИЗМЕРЕНО/verified-live/reproducer marker anywhere — every claim is inferred', ); // 5. README-first — the docs travelled in the same change. // POSITIVE (wave1-scorer-negation): THE acid-A5 defect. crossrt-1-agents-md scored ✓ here over // its own finding "README-first not satisfied — no README was touched": a negation rendered as // a checkmark. A README mention is not a README update. const readme = evidenceLinePositive(qeText + '\n' + manifestText, /README/, NEGATION_QUANTIFIED_RE); add( 'readme-first', 'READMEs updated in the same change', readme !== null ? 'pass' : 'absent', readme ?? 'no README mention in the QE report or the change manifest', ); // 6. The learning loop — Step-0 recall folded in, Step-8 lessons taught. // POSITIVE both halves (wave1-scorer-negation): "recall was not performed" and "no lessons were // taught (dz teach skipped)" both matched the plain regexes and scored the loop as RUN. const recalled = evidenceLinePositive( complexityText + '\n' + allText, /LEARNED_PATTERNS|dz recall|recalled/i, NEGATION_QUANTIFIED_RE, ); const taught = evidenceLinePositive(allText, /lesson[s]? taught|dz teach|taught \(/i, NEGATION_QUANTIFIED_RE); add( 'learning-loop', 'Step-0 recall used; Step-8 lessons taught', recalled !== null && taught !== null ? 'pass' : recalled !== null || taught !== null ? 'partial' : 'absent', recalled !== null && taught !== null ? `${recalled.slice(0, 68)} | ${taught.slice(0, 68)}` : recalled ?? taught ?? 'no recall/teach evidence — the run neither drew on nor fed the learned store', ); // 7. Amendment confirmation — only when amendments EXIST; their absence is not a failure. // Born in the PLAN/ideation: a stray "AM-7" in QE prose (e.g. another feature's test name) // must not conjure the discipline (caught by the very first dogfood run). const plannedAm = [...new Set(planText.match(/AM-\d+/g) ?? [])].sort(); if (plannedAm.length > 0) { // One stray AM-9 in QE must not satisfy AM-1..AM-2 (Codex QE #3): coverage is id-by-id. const covered = plannedAm.filter((id) => qeText.includes(id)); const missing = plannedAm.filter((id) => !qeText.includes(id)); add( 'amendment-confirmation', 'amendments carried into QE with confirmation', missing.length === 0 ? 'pass' : covered.length > 0 ? 'partial' : 'partial', missing.length === 0 ? `all planned amendments reach QE: ${plannedAm.join(', ')}` : `planned ${plannedAm.join(', ')} — QE never mentions ${missing.join(', ')}`, ); } const passed = disciplines.filter((d) => d.verdict === 'pass').length; const total = disciplines.length; const worst = disciplines.filter((d) => d.verdict === 'absent').map((d) => d.id); const summary = `${passed}/${total} disciplines fully evidenced` + (grade !== null ? ` · QE grade ${grade}` : ' · no QE grade') + (worst.length > 0 ? ` · absent: ${worst.join(', ')}` : ''); return { slug, disciplines, qeGrade: grade, passed, total, summary }; } const MARK: Record = { pass: '✓', partial: '◐', absent: '✗' }; export function renderScorecard(card: RunScorecard): string { const out: string[] = []; out.push(`dz score — ${card.slug} (process scorecard; descriptive-only, never a gate)`); out.push(''); for (const d of card.disciplines) { out.push(` ${MARK[d.verdict]} ${d.title}`); out.push(` ${d.evidence}`); } out.push(''); out.push(` ${card.summary}`); return out.join('\n'); }