import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { buildControlFrame } from "./control-frame.ts"; import { createActiveRun, advanceRun, updatePhaseArtifact } from "./state.ts"; import { evaluateToolGateway } from "./tool-gateway.ts"; import { buildDomainPlan } from "./domain-planner.ts"; import { collectMetadataEvidence } from "./metadata-autocollect.ts"; import { hasEvidenceEntry } from "./evidence.ts"; export interface HarnessAuditCategory { name: string; score: number; maxScore: number; passed: string[]; failed: string[]; } export interface HarnessAuditReport { score: number; maxScore: number; categories: HarnessAuditCategory[]; topActions: string[]; } interface AuditCheck { label: string; passed: boolean; action: string; } export function runHarnessAudit(cwd: string): HarnessAuditReport { const root = cwd; const categories = [ category("Tool Coverage", [ fileContains(root, "src/harness/tool-gateway.ts", "isWriteLikeToolCall", "集中写类工具判断,避免散落 toolName 字符串匹配。"), fileContains(root, "extensions/kingdee-harness-tool-events.ts", "beginWriteTransaction", "PI tool_call 写入前必须创建 Write Transaction。"), fileContains(root, "extensions/kingdee-harness-tool-events.ts", "completeWriteTransaction", "PI tool_result 写入后必须执行事务后置校验。"), fileContains(root, "extensions/kingdee-powershell-tool.ts", "powerShellMutationBlockReason", "PowerShell 文件变更绕过必须有阻断函数。"), fileContains(root, "scripts/smoke-harness-tool-gateway-scenario.ts", "unregisteredWriteTransaction", "新增写类工具必须通过 smoke 暴露覆盖边界。"), ]), category("Context Efficiency", [ fileContains(root, "src/harness/context-compiler.ts", "contextBudgetForFocus", "Context Pack 必须按 focus 使用预算。"), fileContains(root, "src/harness/context-compiler.ts", "riskPolicy", "Context Pack 必须包含固定风险策略段。"), fileContains(root, "src/harness/handoff-capsule.ts", "formatRiskPolicyForPrompt", "Handoff Capsule 必须包含固定风险策略段。"), fileContains(root, "src/harness/context-validation.ts", "validateCompiledWorkflowContext", "Context Pack 必须有确定性校验。"), ]), category("Quality Gates", [ fileContains(root, "src/harness/gates.ts", "dataSourceContextBlockReason", "执行阶段必须检查数据源事实/证据。"), fileContains(root, "src/harness/gates.ts", "hasSuccessfulEvidenceEntry", "阶段推进必须检查 evidence index 的成功退出码。"), fileContains(root, "src/harness/action-policy.ts", "sourceWriteBlockReason", "源码写入必须走阶段、事实、证据和 PLAN 门禁。"), fileContains(root, "src/harness/prompt-policy.ts", "PLAN 自由文本不能单独证明", "提示策略必须明确 PLAN 自由文本不能替代 facts/evidence。"), ]), category("Memory Persistence", [ fileContains(root, "src/harness/ledger.ts", "appendLedgerEvent", "关键事件必须进入 append-only ledger。"), fileContains(root, "src/harness/state.ts", "resumeSnapshot", "阻塞问题必须生成可恢复断点。"), fileContains(root, "src/harness/handoff-capsule.ts", "Recent Ledger", "交接包必须包含最近 ledger。"), fileContains(root, "src/harness/transcript-eval.ts", "transcriptScenarioFromLedger", "ledger 必须能生成 replay 场景。"), ]), category("Eval Coverage", [ packageScript(root, "smoke:harness", "package.json 必须暴露 smoke:harness。"), packageScript(root, "smoke:harness-evals", "package.json 必须暴露 smoke:harness-evals。"), packageScript(root, "e2e:agent-demand", "package.json 必须暴露真实需求入口 E2E。"), fileContains(root, "scripts/smoke-harness.ts", "PLAN 自由文本", "smoke 必须覆盖 PLAN 自由文本不能满足事实门禁。"), fileContains(root, "scripts/smoke-harness-evals.ts", "risk-policy-survives-handoff-capsule", "transcript eval 必须覆盖 handoff 风险策略保留。"), fileContains(root, "scripts/e2e-agent-demand.ts", "Agent demand E2E passed", "必须覆盖用户直接输入需求后的 agent prompt 链路。"), ]), category("Security Guardrails", [ fileContains(root, "src/harness/risk-policy.ts", "credential-persistence", "集中风险策略必须包含凭证持久化 hard deny。"), fileContains(root, "src/harness/risk-policy.ts", "write-transaction-bypass", "集中风险策略必须包含写入事务绕过 hard deny。"), fileContains(root, "src/harness/prompt-policy.ts", "禁止在任何仓库文件", "运行时提示必须禁止泄露凭证。"), fileContains(root, "src/harness/evidence.ts", "EvidenceSource", "Evidence index 必须记录来源分级。"), ]), category("Behavioral Architecture", [ behaviorCheck("continue input resolves to advance-phase next action", "普通“继续”必须由 next-action 内核解析为可执行阶段推进。", () => { return withAuditScratch(root, (temp) => { const run = createActiveRun(temp, "苍穹 下推转换工程化改造", "cangqiong", undefined, "normal"); const frame = buildControlFrame(temp, run, "继续"); return frame.nextAction.kind === "advance-phase" && frame.nextAction.executable && frame.nextAction.targetPhase === "spec"; }); }), behaviorCheck("target gate preserves metadata next action", "推进到 execute 失败后,control frame 必须保留目标门禁并指向元数据证据采集。", () => { return withAuditScratch(root, (temp) => { const run = createActiveRun(temp, "苍穹 销售订单保存前校验插件", "cangqiong", undefined, "normal"); advanceRun(temp, run, "spec"); advanceRun(temp, run, "plan"); updatePhaseArtifact(temp, run, "plan", auditPlan("src/main/java/com/example/AuditPlugin.java")); const blocked = advanceRun(temp, run, "execute"); const frame = buildControlFrame(temp, blocked.run, "继续"); return frame.focus === "gate" && frame.nextAction.kind === "collect-metadata-evidence" && frame.nextAction.evidencePath === "evidence/cosmic-metadata.json"; }); }), behaviorCheck("unregistered write-like tool is blocked", "疑似写类工具未登记时必须在 tool gateway 被默认阻断。", () => { const decision = evaluateToolGateway({ cwd: root, toolName: "apply_patch", path: "src/main/java/Demo.java" }); return !decision.allowed && decision.kind === "unregistered-write-tool"; }), behaviorCheck("domain planner classifies integration work", "第三方对接需求必须确定性归类为 external-api/strict,而不是只靠 LLM 语义猜测。", () => { return withAuditScratch(root, (temp) => { const run = createActiveRun(temp, "企业版 第三方 WMS 接口对接,保存后推送订单", "enterprise", undefined, "normal"); const plan = buildDomainPlan(temp, run); return plan.intent === "third-party-integration" && plan.dataAccess === "external-api" && plan.suggestedMode === "normal"; }); }), behaviorCheck("local metadata autocollect writes evidence", "发现本地 FKERNELXML/fdata 后必须能自动采集 evidence,不能只提示 LLM 自己猜工具。", () => { return withAuditScratch(root, (temp) => { const run = createActiveRun(temp, "企业版 采购订单保存校验插件", "enterprise", undefined, "normal"); writeFileSync(join(temp, "enterprise-form-metadata.xml"), auditMetadataXml(), "utf8"); const result = collectMetadataEvidence(temp, run); return result.status === "collected" && hasEvidenceEntry(temp, run, "evidence/data-source.md"); }); }), ]), ]; const maxScore = categories.reduce((sum, item) => sum + item.maxScore, 0); const score = categories.reduce((sum, item) => sum + item.score, 0); return { score, maxScore, categories, topActions: categories.flatMap((item) => item.failed.map((failed) => `${item.name}: ${failed}`)).slice(0, 5), }; } function behaviorCheck(label: string, action: string, run: () => boolean): AuditCheck { try { return { label, passed: run(), action }; } catch { return { label, passed: false, action }; } } function withAuditScratch(root: string, run: (temp: string) => boolean): boolean { const scratch = join(root, ".tmp"); mkdirSync(scratch, { recursive: true }); const temp = mkdtempSync(join(scratch, "kcode-audit-")); try { return run(temp); } finally { rmSync(temp, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 }); } } function auditPlan(path: string): string { return [ "# 实施计划", "", "## 业务数据源上下文", "", "- 触发入口:表单保存前事件", "- 源对象:销售订单 sal_order", "- 目标对象:销售订单 sal_order 当前单据", "- 目标 FormId:sal_order", "- 插件类型和事件:表单插件 BeforeDoOperation 保存前", "- 字段/实体标识:FBillNo, FMaterialId, FQty, FEntity", "- 数据读取写入方式:通过表单模型读取 DynamicObject", "- 数据变化或字段映射:校验 FQty,不修改字段", "- 业务规则和适用条件:当 FQty 小于等于 0 时阻止保存", "- 失败处理:向用户显示数量修正信息并记录校验日志", "- 验收样例:销售订单 SO-001 的 FQty=0 时保存失败,FQty=1 时保存成功", "", "## 必需的金蝶查证项", "", "- 元数据证据:evidence/cosmic-metadata.json", "", "## 允许修改的文件", "", `- ${path}`, "", "## 执行步骤", "", `- [ ] STEP-001:更新 ${path}。`, "", "## TDD / 红绿检查", "", "- 红灯证据:evidence/tdd-red.md", "- 绿灯证据:evidence/tdd-green.md", "- 红绿检查命令或工具:kd_check", "", "## 验证命令", "", ].join("\n"); } function auditMetadataXml(): string { return ` POOrderT_PUR_POORDER采购订单FPOOrder POOrderEntryT_PUR_POORDERENTRY采购订单分录FEntity QtyFEntityFQTY数量FQty `; } export function formatHarnessAuditReport(report: HarnessAuditReport): string { return [ `KCode Harness Audit:${report.score}/${report.maxScore}`, "", ...report.categories.flatMap((category) => [ `## ${category.name}:${category.score}/${category.maxScore}`, ...category.passed.map((item) => `- PASS:${item}`), ...category.failed.map((item) => `- FAIL:${item}`), "", ]), "## Top Actions", report.topActions.length > 0 ? report.topActions.map((item, index) => `${index + 1}. ${item}`).join("\n") : "无。", ].join("\n"); } function category(name: string, checks: AuditCheck[]): HarnessAuditCategory { return { name, score: checks.filter((check) => check.passed).length, maxScore: checks.length, passed: checks.filter((check) => check.passed).map((check) => check.label), failed: checks.filter((check) => !check.passed).map((check) => check.action), }; } function fileContains(root: string, path: string, pattern: string, action: string): AuditCheck { const absolute = join(root, path); const content = existsSync(absolute) ? readFileSync(absolute, "utf8") : ""; return { label: `${path} contains ${pattern}`, passed: content.includes(pattern), action }; } function packageScript(root: string, script: string, action: string): AuditCheck { const packageJsonPath = join(root, "package.json"); const content = existsSync(packageJsonPath) ? readFileSync(packageJsonPath, "utf8") : "{}"; return { label: `package.json script ${script}`, passed: content.includes(`"${script}"`), action }; }