#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SCRIPT_PATH="$SCRIPT_DIR/$(basename "$0")"
SUMMARY_CORE_PATH="$SCRIPT_DIR/pi67-xtalpi-smoke-artifact-core.cjs"
OUT_DIR="${OUT_DIR:-$HOME/tmp/xtalpi-pi-tools-smoke}"
FORMAT="text"
PROFILE=""
LATEST_ONLY="0"
RUN_ID=""
HISTORY_LIMIT=""
TREND_GATE_LIMIT=""
DRIFT_LIMIT=""
RETENTION_REPORT="0"
RETENTION_POLICY_OPTION_USED="0"
COMPARE_BASE_RUN_ID=""
COMPARE_HEAD_RUN_ID=""
FAIL_ON_RECOVERY_INCREASE="0"
MAX_RECOVERY_CASE_RUNS=""
EXPECT_CASES=""
EXPECT_CASE_NAMES=""
MAX_ERRORS="0"
MAX_EMPTY_ASSISTANT_ENDS=""
MAX_RAW_TOOL_MARKUP_FINAL_ANSWERS=""
MAX_RECOVERIES=""
MAX_RECOVERY_RATE=""
MAX_REQUEST_LATENCY_MS=""
MAX_SLOW_REQUESTS=""
REQUIRE_TOOL_SELECTION_REASON_CODES=""
REQUIRE_SELECTED_TOOL_SELECTION_REASON_CODES=""
REQUIRE_OMITTED_TOOL_SELECTION_REASON_CODES=""
FORBID_TOOL_SELECTION_REASON_CODES=""
FORBID_SELECTED_TOOL_SELECTION_REASON_CODES=""
FORBID_OMITTED_TOOL_SELECTION_REASON_CODES=""
RUN_KIND_FILTER=""
REQUIRE_RUN_KIND=""
REQUIRE_STABLE_RUNTIME_FINGERPRINT="0"
REQUIRE_STABLE_RUNTIME_BOUNDS="0"
KEEP_FULL_SUITE="10"
KEEP_TARGETED="10"
KEEP_PREFLIGHT_FAILED="10"
KEEP_EMPTY="5"
usage() {
cat <<'EOF'
Usage: pi67-xtalpi-pi-tools-debug-summary.sh [--json] [--latest|--run-id RUN_ID] [options] [OUT_DIR]
Summarize xtalpi-pi-tools live smoke artifacts:
- *.debug.jsonl provider telemetry
- matching *.jsonl Pi event streams, when present
Selection:
--latest summarize the newest run id
--run-id RUN_ID summarize one exact smoke run, e.g. 20260702-144643 or 20260702-144643-12345
--history N show newest N persisted *-summary.json smoke runs
--trend-gate N gate newest N persisted smoke summaries
--drift N summarize provider/runtime drift across newest N persisted smoke summaries
--retention-report report read-only artifact retention/hygiene recommendations
--compare BASE_RUN HEAD_RUN compare two persisted smoke summaries
--run-kind LIST for --history/--trend-gate/--drift, filter persisted summaries by runKind before selecting newest N
--require-run-kind LIST require selected run(s) to have one of the comma-separated runKind values
Gate options:
--profile full-suite-strict|full-suite-runtime-strict|full-suite-ranking-strict
apply built-in trend gate defaults
--expect-cases N require case count for --latest/--run-id and every --trend-gate run
--expect-case-names LIST require exact comma-separated case names for --latest/--run-id and --trend-gate
--max-errors N default: 0
--max-empty-assistant-ends N
--max-raw-tool-markup-final-answers N
--max-tool-envelope-final-answers N alias for --max-raw-tool-markup-final-answers
--max-recoveries N
--max-recovery-rate N recoveries / turns
--max-request-latency-ms N fail if selected run/case max request latency exceeds N
--max-slow-requests N fail if selected run/case slow request count exceeds N
--require-tool-selection-reason-codes LIST
require every selected run to include all comma-separated reason codes
--require-selected-tool-selection-reason-codes LIST
require every selected run's selected-tool reason codes to include LIST
--require-omitted-tool-selection-reason-codes LIST
require every selected run's omitted-tool reason codes to include LIST
--forbid-tool-selection-reason-codes LIST
fail if any selected run includes any comma-separated reason code
--forbid-selected-tool-selection-reason-codes LIST
fail if selected-tool reason codes include LIST
--forbid-omitted-tool-selection-reason-codes LIST
fail if omitted-tool reason codes include LIST
--fail-on-recovery-increase fail if the newest run has more recoveries or a higher recovery rate
--max-recovery-case-runs N fail if one case has recoveries in more than N selected runs
--require-stable-runtime require stable runtime fingerprint and runtime bounds across --trend-gate runs
--require-stable-runtime-fingerprint
require stable runtime fingerprint across --trend-gate runs
--require-stable-runtime-bounds require stable runtime bounds across --trend-gate runs
Retention report options:
--keep-full-suite N retain newest N full-suite runs before suggesting archive; default: 10
--keep-targeted N retain newest N targeted runs before suggesting archive; default: 10
--keep-preflight-failed N retain newest N preflight-failed runs before suggesting archive; default: 10
--keep-empty N retain newest N empty runs before suggesting archive; default: 5
Default OUT_DIR:
$HOME/tmp/xtalpi-pi-tools-smoke
EOF
}
FULL_SUITE_CASE_NAMES="no-tool,bash,read,bash-read,web-read,plan-mode-contract,plan-mode-accepted-continuation,read-enoent-recovery,tool-selection-clipping,tool-selection-continuation,until-done-continuation,tool-result-injection"
FULL_SUITE_REQUIRED_TOOL_SELECTION_REASON_CODES="core_tool,prompt_path_file"
FULL_SUITE_REQUIRED_SELECTED_TOOL_SELECTION_REASON_CODES="core_tool,prompt_path_file"
FULL_SUITE_REQUIRED_OMITTED_TOOL_SELECTION_REASON_CODES="core_tool"
FULL_SUITE_FORBIDDEN_TOOL_SELECTION_REASON_CODES="prompt_tool_exclusive"
apply_profile_defaults() {
case "$PROFILE" in
"")
;;
full-suite-strict|full-suite-runtime-strict|full-suite-ranking-strict)
EXPECT_CASES="${EXPECT_CASES:-12}"
EXPECT_CASE_NAMES="${EXPECT_CASE_NAMES:-$FULL_SUITE_CASE_NAMES}"
RUN_KIND_FILTER="${RUN_KIND_FILTER:-full-suite}"
REQUIRE_RUN_KIND="${REQUIRE_RUN_KIND:-full-suite}"
MAX_EMPTY_ASSISTANT_ENDS="${MAX_EMPTY_ASSISTANT_ENDS:-0}"
MAX_RAW_TOOL_MARKUP_FINAL_ANSWERS="${MAX_RAW_TOOL_MARKUP_FINAL_ANSWERS:-0}"
MAX_RECOVERIES="${MAX_RECOVERIES:-2}"
MAX_RECOVERY_RATE="${MAX_RECOVERY_RATE:-0.15}"
MAX_RECOVERY_CASE_RUNS="${MAX_RECOVERY_CASE_RUNS:-3}"
if [ "$PROFILE" = "full-suite-runtime-strict" ]; then
REQUIRE_STABLE_RUNTIME_FINGERPRINT="1"
REQUIRE_STABLE_RUNTIME_BOUNDS="1"
fi
if [ "$PROFILE" = "full-suite-ranking-strict" ]; then
REQUIRE_TOOL_SELECTION_REASON_CODES="${REQUIRE_TOOL_SELECTION_REASON_CODES:-$FULL_SUITE_REQUIRED_TOOL_SELECTION_REASON_CODES}"
REQUIRE_SELECTED_TOOL_SELECTION_REASON_CODES="${REQUIRE_SELECTED_TOOL_SELECTION_REASON_CODES:-$FULL_SUITE_REQUIRED_SELECTED_TOOL_SELECTION_REASON_CODES}"
REQUIRE_OMITTED_TOOL_SELECTION_REASON_CODES="${REQUIRE_OMITTED_TOOL_SELECTION_REASON_CODES:-$FULL_SUITE_REQUIRED_OMITTED_TOOL_SELECTION_REASON_CODES}"
FORBID_TOOL_SELECTION_REASON_CODES="${FORBID_TOOL_SELECTION_REASON_CODES:-$FULL_SUITE_FORBIDDEN_TOOL_SELECTION_REASON_CODES}"
fi
;;
*)
echo "xtalpi-pi-tools debug summary: unknown --profile '$PROFILE' (supported: full-suite-strict, full-suite-runtime-strict, full-suite-ranking-strict)" >&2
exit 2
;;
esac
}
run_self_test() {
local tmp_dir
local output
tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/pi67-xtalpi-debug-summary-self-test.XXXXXX")"
trap "rm -rf '$tmp_dir'" EXIT
node - "$tmp_dir" <<'NODE'
const fs = require("node:fs");
const crypto = require("node:crypto");
const path = require("node:path");
const root = process.argv[2];
function ensureDir(name) {
const dir = path.join(root, name);
fs.mkdirSync(dir, { recursive: true });
return dir;
}
function writeJsonl(dir, file, events) {
fs.writeFileSync(path.join(dir, file), events.map((event) => JSON.stringify(event)).join("\n") + "\n");
}
function writeCase(
dir,
runId,
name,
{ finalText = "normal final answer", debugEvents = [], toolNames = [], lifecycle } = {},
) {
writeJsonl(dir, `${runId}-${name}.jsonl`, [
...toolNames.map((toolName) => ({ type: "tool_execution_start", toolName, args: {} })),
{
type: "agent_end",
messages: [
{
role: "assistant",
content: [{ type: "text", text: finalText }],
stopReason: "stop",
},
],
},
]);
writeJsonl(dir, `${runId}-${name}.debug.jsonl`, debugEvents.length ? debugEvents : [
{
schema: "xtalpi-pi-tools.debug.v1",
event: "turn.start",
event_category: "turn",
selected_tool_count: toolNames.length,
},
]);
if (lifecycle) {
fs.writeFileSync(path.join(dir, `${runId}-${name}.lifecycle.json`), `${JSON.stringify({
schema: "xtalpi-pi-tools.smoke-lifecycle.v1",
caseName: name,
...lifecycle,
}, null, 2)}\n`);
}
}
const clean = ensureDir("clean");
writeCase(clean, "20260702-000001", "clean", {
toolNames: ["read"],
debugEvents: [
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:00:00.000Z",
event: "turn.start",
event_category: "turn",
selected_tool_count: 1,
tool_selection_clipped: true,
tool_selection_omitted_count: 2,
tool_selection_valid_count: 3,
tool_selection_prompt_source: "recent_user_continuation",
tool_selection_prompt_chars: 128,
tool_selection_user_messages: 2,
data: {
toolSelectionClipped: true,
toolSelectionOmittedCount: 2,
toolSelectionValidCount: 3,
toolSelectionPromptSource: "recent_user_continuation",
toolSelectionPromptChars: 128,
toolSelectionUserMessageCount: 2,
toolSelectionSummary: {
schema: "xtalpi-pi-tools.tool-selection.v1",
validToolCount: 3,
omittedToolCount: 2,
selected: [
{
name: "read",
index: 0,
score: 160,
selected: true,
reasonCodes: ["prompt_tool_exclusive", "prompt_tool_name"],
},
],
omitted: [
{
name: "hidden_admin",
index: 1,
score: 0,
selected: false,
reasonCodes: ["prompt_tool_forbidden"],
},
],
},
},
},
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:00:00.000Z",
event: "request",
event_category: "request",
},
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:00:01.500Z",
event: "response",
event_category: "response",
},
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:00:02.000Z",
event: "request",
event_category: "request",
},
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:01:04.000Z",
event: "response",
event_category: "response",
},
{
schema: "xtalpi-pi-tools.debug.v1",
event: "tool_call",
event_category: "tool_call",
selected_tool_count: 1,
argument_validation_warning_count: 1,
argument_validation_warning_codes: ["pattern_nested_quantifier"],
data: {
argumentValidationWarningCount: 1,
argumentValidationWarningCodes: ["pattern_nested_quantifier"],
argumentValidationWarnings: [
{
code: "pattern_nested_quantifier",
path: "arguments.value",
patternChars: 7,
inputChars: 2049,
},
],
},
},
],
});
const raw = ensureDir("raw");
writeCase(raw, "20260702-000002", "raw", {
toolNames: ["read"],
finalText: "\nid: call_1\nname: read\n",
});
const malformed = ensureDir("malformed");
writeCase(malformed, "20260702-000004", "malformed", {
toolNames: ["read"],
finalText: "\nid: call_1\nname: read\narguments_json: {\"path\":\"package.json\"}\n",
});
const recovery = ensureDir("recovery");
writeCase(recovery, "20260702-000003", "recovering", {
debugEvents: [
{
schema: "xtalpi-pi-tools.debug.v1",
event: "turn.start",
event_category: "turn",
selected_tool_count: 1,
},
{
schema: "xtalpi-pi-tools.debug.v1",
event: "recovery.raw_protocol_markup",
event_category: "recovery",
selected_tool_count: 1,
},
],
});
const lifecycle = ensureDir("lifecycle");
writeCase(lifecycle, "20260702-000006", "timed-out-after-agent-end", {
toolNames: ["web_fetch", "read"],
lifecycle: {
exitStatus: 124,
elapsedSeconds: 42,
caseTimeoutSeconds: 40,
timedOutByWatchdog: true,
agentEndSeenDuringRun: true,
agentEndElapsedSeconds: 12,
postAgentEndLingerSeconds: 30,
},
});
const providerError = ensureDir("provider-error");
writeCase(providerError, "20260702-000007", "http-429", {
debugEvents: [
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:00:00.000Z",
event: "request",
event_category: "request",
},
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:01:01.000Z",
event: "error.provider",
event_category: "error",
event_kind: "provider",
provider: "xtalpi-pi-tools",
model: "deepseek-v4-pro",
error_code: "http_429",
error_category: "rate_limit",
retryable: true,
http_status: 429,
data: {
errorCode: "http_429",
errorCategory: "rate_limit",
retryable: true,
httpStatus: 429,
},
},
],
});
const history = ensureDir("history");
function writeSummary(
runId,
{
dir = history,
ok = true,
failures = 0,
recoveries = 0,
raw = 0,
errors = 0,
cases,
totalCases = 5,
selectedCases,
provider = "xtalpi-pi-tools",
model = "deepseek-v4-pro",
caseTimeoutSeconds = 180,
requestTimeoutMs = 180000,
maxOutputTokens = 1024,
providerHealth,
runtimeFingerprint,
requestLatencyTotals,
toolSelectionReasonCodes,
selectedToolSelectionReasonCodes,
omittedToolSelectionReasonCodes,
} = {},
) {
const defaultRuntimeFingerprint = runtimeFingerprint || {
protocolVersions: ["xtalpi-pi-tools.json-action.v1"],
selectedToolNameHashes: ["3316348dbadfb7b1"],
selectedToolNames: ["read"],
maxTools: [24],
toolSelectionClipped: [false],
toolSelectionOmittedCount: [0],
toolSelectionValidCount: [1],
toolSelectionPromptSources: ["latest_user"],
toolSelectionPromptChars: [56],
toolSelectionUserMessageCount: [1],
maxToolResultChars: [20000],
maxOutputTokens: [maxOutputTokens],
requestTimeoutMs: [requestTimeoutMs],
maxEmptyRetries: [2],
maxRepairRetries: [2],
maxTotalRecoveries: [4],
};
const caseItems = cases || [
{
runId,
caseName: "read",
turns: 2,
toolCalls: 1,
recoveries,
emptyAssistantEnds: 0,
rawToolMarkupFinalAnswer: raw > 0,
toolEnvelopeFinalAnswer: false,
errors,
finalTextChars: 42,
piToolStarts: ["read"],
},
];
const caseItemsWithRuntime = caseItems.map((item) => item.runtimeFingerprint
? item
: { ...item, runtimeFingerprint: defaultRuntimeFingerprint });
const defaultToolSelectionReasonCodes = toolSelectionReasonCodes || {};
const defaultSelectedToolSelectionReasonCodes = selectedToolSelectionReasonCodes || {};
const defaultOmittedToolSelectionReasonCodes = omittedToolSelectionReasonCodes || {};
const selectedCaseNames = selectedCases || caseItems.map((item) => item.caseName);
const normalizedCases = [...new Set(selectedCaseNames)].sort();
const canonical = normalizedCases.join(",");
const caseSet = {
schema: "xtalpi-pi-tools.case-set.v1",
selectedCases: selectedCaseNames,
normalizedCases,
count: normalizedCases.length,
canonical,
sha256: crypto.createHash("sha256").update(canonical).digest("hex"),
};
fs.writeFileSync(path.join(dir, `${runId}-summary.json`), `${JSON.stringify({
schema: "xtalpi-pi-tools.smoke-summary.v1",
createdAt: "2026-07-02T00:00:00.000Z",
provider,
model,
stamp: runId,
runId,
outDir: dir,
selectedCases: selectedCaseNames,
caseSet,
caseTimeoutSeconds,
requestTimeoutMs,
maxOutputTokens,
providerHealth: providerHealth || {
schema: "xtalpi-pi-tools.provider-health.v1",
provider,
model,
ok: true,
timeoutMs: 30000,
elapsedMs: 100,
attemptsConfigured: 2,
attemptCount: 1,
retryCount: 0,
retryDelayMs: 1000,
httpStatus: 200,
responseModel: model,
},
failures,
debugSummaryStatus: 0,
ok,
debugSummary: {
outDir: dir,
latestOnly: false,
runId,
gateFailures: ok ? [] : ["fixture failure"],
totals: {
cases: totalCases,
debugEvents: 20,
turns: 10,
toolCalls: 6,
recoveries,
recoveryRate: recoveries / 10,
emptyAssistantEnds: 0,
rawToolMarkupFinalAnswers: raw,
toolEnvelopeFinalAnswers: 0,
piToolStarts: 6,
errors,
toolSelectionReasonCodes: defaultToolSelectionReasonCodes,
selectedToolSelectionReasonCodes: defaultSelectedToolSelectionReasonCodes,
omittedToolSelectionReasonCodes: defaultOmittedToolSelectionReasonCodes,
...(requestLatencyTotals ? {
requestCount: requestLatencyTotals.requestCount,
requestLatencyMsMin: requestLatencyTotals.requestLatencyMsMin,
requestLatencyMsMax: requestLatencyTotals.requestLatencyMsMax,
requestLatencyMsAvg: requestLatencyTotals.requestLatencyMsAvg,
slowRequestCount: requestLatencyTotals.slowRequestCount,
slowRequestThresholdMs: requestLatencyTotals.slowRequestThresholdMs,
} : {}),
},
cases: caseItemsWithRuntime,
},
}, null, 2)}\n`);
}
writeSummary("20260702-000001", { ok: true, failures: 0, recoveries: 0 });
writeSummary("20260702-000002", {
ok: true,
failures: 0,
recoveries: 2,
requestLatencyTotals: {
requestCount: 2,
requestLatencyMsMin: 1500,
requestLatencyMsMax: 62000,
requestLatencyMsAvg: 31750,
slowRequestCount: 1,
slowRequestThresholdMs: 60000,
},
});
writeSummary("20260702-000003", { ok: false, failures: 1, recoveries: 1, raw: 1 });
fs.writeFileSync(path.join(history, "20260702-000004-debug-summary.json"), "{}\n");
const trend = ensureDir("trend");
const fullSuiteCases = [
"no-tool",
"bash",
"read",
"bash-read",
"web-read",
"plan-mode-contract",
"plan-mode-accepted-continuation",
"read-enoent-recovery",
"tool-selection-clipping",
"tool-selection-continuation",
"until-done-continuation",
"tool-result-injection",
];
const fullSuiteReasonCodeTotals = {
toolSelectionReasonCodes: { core_tool: 6, prompt_path_file: 2, prompt_tool_name: 4 },
selectedToolSelectionReasonCodes: { core_tool: 4, prompt_path_file: 2, prompt_tool_name: 3 },
omittedToolSelectionReasonCodes: { core_tool: 2, prompt_tool_name: 1 },
};
writeSummary("20260702-000001", {
dir: trend,
ok: true,
failures: 0,
recoveries: 0,
selectedCases: ["no-tool", "bash", "read", "bash-read", "web-read"],
cases: [
{
runId: "20260702-000001",
caseName: "web-read",
turns: 4,
toolCalls: 3,
recoveries: 0,
emptyAssistantEnds: 0,
rawToolMarkupFinalAnswer: false,
toolEnvelopeFinalAnswer: false,
errors: 0,
piToolStarts: ["web_fetch", "read", "read"],
},
],
});
writeSummary("20260702-000002", {
dir: trend,
ok: true,
failures: 0,
recoveries: 1,
selectedCases: ["no-tool", "bash", "read", "bash-read", "web-read"],
cases: [
{
runId: "20260702-000002",
caseName: "web-read",
turns: 4,
toolCalls: 3,
recoveries: 1,
emptyAssistantEnds: 0,
rawToolMarkupFinalAnswer: false,
toolEnvelopeFinalAnswer: false,
errors: 0,
piToolStarts: ["web_fetch", "read", "read"],
},
],
});
const fullSuiteTrend = ensureDir("full-suite-trend");
writeSummary("20260702-000001", {
dir: fullSuiteTrend,
ok: true,
failures: 0,
recoveries: 0,
totalCases: fullSuiteCases.length,
selectedCases: fullSuiteCases,
...fullSuiteReasonCodeTotals,
});
writeSummary("20260702-000002", {
dir: fullSuiteTrend,
ok: true,
failures: 0,
recoveries: 0,
totalCases: fullSuiteCases.length,
selectedCases: fullSuiteCases,
...fullSuiteReasonCodeTotals,
});
const mixedFullSuiteTrend = ensureDir("mixed-full-suite-trend");
writeSummary("20260702-000001", {
dir: mixedFullSuiteTrend,
ok: true,
failures: 0,
recoveries: 0,
totalCases: fullSuiteCases.length,
selectedCases: fullSuiteCases,
...fullSuiteReasonCodeTotals,
});
writeSummary("20260702-000002", {
dir: mixedFullSuiteTrend,
ok: true,
failures: 0,
recoveries: 0,
totalCases: fullSuiteCases.length,
selectedCases: fullSuiteCases,
...fullSuiteReasonCodeTotals,
});
writeSummary("20260702-000003", {
dir: mixedFullSuiteTrend,
ok: true,
failures: 0,
recoveries: 0,
totalCases: 1,
selectedCases: ["web-read"],
});
writeSummary("20260702-000004", {
dir: mixedFullSuiteTrend,
ok: true,
failures: 0,
recoveries: 0,
totalCases: fullSuiteCases.length,
selectedCases: fullSuiteCases,
...fullSuiteReasonCodeTotals,
});
writeCase(mixedFullSuiteTrend, "20260702-000005", "orphan", {
toolNames: ["read"],
});
fs.writeFileSync(path.join(mixedFullSuiteTrend, "manual-note.txt"), "manual artifact note\n");
const driftHistory = ensureDir("drift");
writeSummary("20260702-000001", {
dir: driftHistory,
ok: true,
failures: 0,
recoveries: 0,
totalCases: fullSuiteCases.length,
selectedCases: fullSuiteCases,
...fullSuiteReasonCodeTotals,
});
writeSummary("20260702-000002", {
dir: driftHistory,
ok: true,
failures: 0,
recoveries: 0,
totalCases: fullSuiteCases.length,
selectedCases: fullSuiteCases,
maxOutputTokens: 2048,
...fullSuiteReasonCodeTotals,
runtimeFingerprint: {
protocolVersions: ["xtalpi-pi-tools.json-action.v1"],
selectedToolNameHashes: ["3316348dbadfb7b1"],
selectedToolNames: ["read"],
maxTools: [24],
toolSelectionClipped: [false],
toolSelectionOmittedCount: [0],
toolSelectionValidCount: [1],
toolSelectionPromptSources: ["latest_user"],
toolSelectionPromptChars: [56],
toolSelectionUserMessageCount: [1],
maxToolResultChars: [20000],
maxOutputTokens: [2048],
requestTimeoutMs: [180000],
maxEmptyRetries: [2],
maxRepairRetries: [2],
maxTotalRecoveries: [4],
},
});
writeSummary("20260702-000003", {
dir: driftHistory,
ok: true,
failures: 0,
recoveries: 1,
totalCases: 1,
selectedCases: ["web-read"],
});
const rankingDriftTrend = ensureDir("ranking-drift-trend");
writeSummary("20260702-000001", {
dir: rankingDriftTrend,
ok: true,
failures: 0,
recoveries: 0,
totalCases: fullSuiteCases.length,
selectedCases: fullSuiteCases,
...fullSuiteReasonCodeTotals,
});
writeSummary("20260702-000002", {
dir: rankingDriftTrend,
ok: true,
failures: 0,
recoveries: 0,
totalCases: fullSuiteCases.length,
selectedCases: fullSuiteCases,
toolSelectionReasonCodes: { core_tool: 6, prompt_tool_exclusive: 1 },
selectedToolSelectionReasonCodes: { core_tool: 4 },
omittedToolSelectionReasonCodes: {},
});
const subsetTrend = ensureDir("subset-trend");
writeSummary("20260702-000001", {
dir: subsetTrend,
ok: true,
failures: 0,
recoveries: 0,
totalCases: 1,
});
writeSummary("20260702-000002", {
dir: subsetTrend,
ok: true,
failures: 0,
recoveries: 0,
totalCases: 1,
});
const shortTrend = ensureDir("short-trend");
writeSummary("20260702-000001", {
dir: shortTrend,
ok: true,
failures: 0,
recoveries: 0,
});
const latencyTrend = ensureDir("latency-trend");
writeSummary("20260702-000001", {
dir: latencyTrend,
ok: true,
failures: 0,
recoveries: 0,
requestLatencyTotals: {
requestCount: 1,
requestLatencyMsMin: 1500,
requestLatencyMsMax: 1500,
requestLatencyMsAvg: 1500,
slowRequestCount: 0,
slowRequestThresholdMs: 60000,
},
});
writeSummary("20260702-000002", {
dir: latencyTrend,
ok: true,
failures: 0,
recoveries: 0,
requestLatencyTotals: {
requestCount: 2,
requestLatencyMsMin: 2000,
requestLatencyMsMax: 62000,
requestLatencyMsAvg: 32000,
slowRequestCount: 1,
slowRequestThresholdMs: 60000,
},
});
const latencyBackfill = ensureDir("latency-backfill");
writeSummary("20260702-000001", {
dir: latencyBackfill,
ok: true,
failures: 0,
recoveries: 0,
totalCases: 1,
selectedCases: ["read"],
});
writeCase(latencyBackfill, "20260702-000001", "read", {
toolNames: ["read"],
debugEvents: [
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:00:00.000Z",
event: "request",
event_category: "request",
},
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:00:01.500Z",
event: "response",
event_category: "response",
},
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:00:02.000Z",
event: "request",
event_category: "request",
},
{
schema: "xtalpi-pi-tools.debug.v1",
ts: "2026-07-02T00:01:04.000Z",
event: "response",
event_category: "response",
},
],
});
NODE
if ! output="$("$SCRIPT_PATH" --run-id 20260702-000001 --expect-cases 1 --expect-case-names clean --max-errors 0 --max-empty-assistant-ends 0 --max-raw-tool-markup-final-answers 0 --max-recoveries 0 "$tmp_dir/clean" 2>&1)"; then
echo "$output"
return 1
fi
if [[ "$output" != *"tool_selection_clipped=true"* || "$output" != *"tool_selection_omitted=2-2"* || "$output" != *"tool_selection_valid=3-3"* || "$output" != *"tool_selection_prompt_source=recent_user_continuation"* || "$output" != *"tool_selection_reason_codes="* || "$output" != *"prompt_tool_exclusive"* || "$output" != *"prompt_tool_forbidden"* || "$output" != *"argument_validation_warnings=1"* || "$output" != *"pattern_nested_quantifier"* ]]; then
echo "clean fixture output did not expose bounded tool-selection or argument-validation diagnostics"
echo "$output"
return 1
fi
if [[ "$output" != *"request_latency_ms=62000/31750/2"* || "$output" != *"slow_requests=1"* || "$output" != *"slow_request_threshold_ms=60000"* ]]; then
echo "clean fixture output did not expose request latency diagnostics"
echo "$output"
return 1
fi
if ! output="$("$SCRIPT_PATH" --run-id 20260702-000001 --require-tool-selection-reason-codes prompt_tool_exclusive,prompt_tool_forbidden --require-selected-tool-selection-reason-codes prompt_tool_exclusive --require-omitted-tool-selection-reason-codes prompt_tool_forbidden --forbid-tool-selection-reason-codes nonexistent "$tmp_dir/clean" 2>&1)"; then
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --run-id 20260702-000001 --forbid-tool-selection-reason-codes prompt_tool_forbidden "$tmp_dir/clean" 2>&1)"; then
echo "expected direct reason-code gate to fail"
echo "$output"
return 1
fi
if [[ "$output" != *"tool_selection_reason_codes"* || "$output" != *"prompt_tool_forbidden"* ]]; then
echo "direct reason-code gate failure did not expose forbidden reason code"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --run-id 20260702-000001 --max-request-latency-ms 60000 "$tmp_dir/clean" 2>&1)"; then
echo "expected direct request latency gate to fail"
echo "$output"
return 1
fi
if [[ "$output" != *"expected request_latency_ms_max<=60000, got 62000"* ]]; then
echo "direct request latency gate did not expose expected failure"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --run-id 20260702-000001 --max-slow-requests 0 "$tmp_dir/clean" 2>&1)"; then
echo "expected direct slow request gate to fail"
echo "$output"
return 1
fi
if [[ "$output" != *"expected slow_requests<=0, got 1"* ]]; then
echo "direct slow request gate did not expose expected failure"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --run-id 20260702-000001 --expect-case-names read "$tmp_dir/clean" 2>&1)"; then
echo "expected direct case-name gate to fail"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --latest --expect-cases 1 --max-errors 0 --max-empty-assistant-ends 0 --max-raw-tool-markup-final-answers 0 --max-recoveries 0 "$tmp_dir/raw" 2>&1)"; then
echo "expected raw final-answer fixture to fail"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --latest --expect-cases 1 --max-errors 0 --max-empty-assistant-ends 0 --max-raw-tool-markup-final-answers 0 --max-recoveries 0 "$tmp_dir/malformed" 2>&1)"; then
echo "expected malformed final-answer fixture to fail"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --latest --expect-cases 1 --max-errors 0 --max-empty-assistant-ends 0 --max-raw-tool-markup-final-answers 0 --max-recoveries 0 "$tmp_dir/neutral-history" 2>&1)"; then
echo "expected neutral history final-answer fixture to fail"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --latest --expect-cases 1 --max-errors 0 --max-empty-assistant-ends 0 --max-raw-tool-markup-final-answers 0 --max-recoveries 0 "$tmp_dir/recovery" 2>&1)"; then
echo "expected recovery-threshold fixture to fail"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --latest --expect-cases 1 --max-errors 0 --max-empty-assistant-ends 0 --max-raw-tool-markup-final-answers 0 --max-recoveries 0 "$tmp_dir/lifecycle" 2>&1)"; then
echo "expected lifecycle timeout fixture to fail"
echo "$output"
return 1
fi
if [[ "$output" != *"process_lifecycle_failures"* || "$output" != *"timed_out_after_agent_end"* ]]; then
echo "lifecycle timeout output did not expose process lifecycle diagnostics"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --latest --expect-cases 1 --max-errors 0 --max-empty-assistant-ends 0 --max-raw-tool-markup-final-answers 0 --max-recoveries 0 "$tmp_dir/provider-error" 2>&1)"; then
echo "expected provider error fixture to fail"
echo "$output"
return 1
fi
if [[ "$output" != *"provider_errors"* || "$output" != *"provider_error_codes"* || "$output" != *"http_429"* ]]; then
echo "provider error output did not expose structured provider diagnostics"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --latest --expect-cases 2 "$tmp_dir/clean" 2>&1)"; then
echo "expected case-count fixture to fail"
echo "$output"
return 1
fi
local history_json="$tmp_dir/history-output.json"
if ! output="$("$SCRIPT_PATH" --history 2 --json "$tmp_dir/history" >"$history_json" 2>&1)"; then
echo "$output"
return 1
fi
if ! node - "$history_json" <<'NODE'; then
const fs = require("node:fs");
const file = process.argv[2];
const data = JSON.parse(fs.readFileSync(file, "utf8"));
function assert(condition, message) {
if (!condition) throw new Error(message);
}
assert(data.schema === "xtalpi-pi-tools.smoke-history.v1", "unexpected history schema");
assert(data.totalArtifacts === 3, "debug-summary artifact should not be counted as a smoke summary");
assert(data.runs.length === 2, "history limit did not select two runs");
assert(data.runs[0].runId === "20260702-000003", "newest run should be first");
assert(data.runs[1].runId === "20260702-000002", "second newest run should be second");
assert(data.runs[0].runKind === "targeted", "history should classify subset fixture as targeted");
assert(data.runs[0].ok === false && data.runs[0].failures === 1, "failed run was not visible");
assert(data.runs[0].rawToolMarkupFinalAnswers === 1, "raw final-answer count was not preserved");
assert(data.runs[1].recoveries === 2, "recovery run was not visible");
assert(data.runs[1].caseSet.schema === "xtalpi-pi-tools.case-set.v1", "case set schema missing");
assert(data.runs[1].caseSet.canonical === "read", "case set canonical should use stable case names");
assert(/^[a-f0-9]{64}$/.test(data.runs[1].caseSet.sha256), "case set hash missing");
assert(data.runs[1].requestCount === 2, "persisted request count was not preserved");
assert(data.runs[1].requestLatencyMsMax === 62000, "persisted request latency max was not preserved");
assert(data.runs[1].slowRequestCount === 1, "persisted slow request count was not preserved");
NODE
return 1
fi
local latency_backfill_json="$tmp_dir/latency-backfill-output.json"
if ! output="$("$SCRIPT_PATH" --history 1 --json "$tmp_dir/latency-backfill" >"$latency_backfill_json" 2>&1)"; then
echo "$output"
return 1
fi
if ! node - "$latency_backfill_json" <<'NODE'; then
const fs = require("node:fs");
const file = process.argv[2];
const data = JSON.parse(fs.readFileSync(file, "utf8"));
function assert(condition, message) {
if (!condition) throw new Error(message);
}
const run = data.runs[0];
assert(run.requestCount === 2, "legacy persisted summary latency was not backfilled from debug JSONL");
assert(run.requestLatencyMsMin === 1500, "backfilled request latency min was wrong");
assert(run.requestLatencyMsMax === 62000, "backfilled request latency max was wrong");
assert(run.requestLatencyMsAvg === 31750, "backfilled request latency avg was wrong");
assert(run.slowRequestCount === 1, "backfilled slow request count was wrong");
NODE
return 1
fi
if ! output="$("$SCRIPT_PATH" --history 2 "$tmp_dir/history" 2>&1)"; then
echo "$output"
return 1
fi
if [[ "$output" != *"20260702-000003"* || "$output" != *"20260702-000002"* || "$output" != *"run_kind=targeted"* || "$output" != *"case_set_sha256="* || "$output" == *"20260702-000001"* ]]; then
echo "history text output did not show only the newest two runs"
echo "$output"
return 1
fi
if ! output="$("$SCRIPT_PATH" --history 2 --run-kind full-suite "$tmp_dir/mixed-full-suite-trend" 2>&1)"; then
echo "$output"
return 1
fi
if [[ "$output" != *"run_kind_filter=full-suite"* || "$output" != *"filtered_out_artifacts=1"* || "$output" != *"20260702-000004"* || "$output" != *"20260702-000002"* || "$output" == *"20260702-000003"* ]]; then
echo "runKind-filtered history did not select newest full-suite runs"
echo "$output"
return 1
fi
local drift_json="$tmp_dir/drift-output.json"
if ! output="$("$SCRIPT_PATH" --drift 2 --run-kind full-suite --json "$tmp_dir/drift" >"$drift_json" 2>&1)"; then
echo "$output"
return 1
fi
if ! node - "$drift_json" <<'NODE'; then
const fs = require("node:fs");
const data = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
function assert(condition, message) {
if (!condition) throw new Error(message);
}
assert(data.schema === "xtalpi-pi-tools.smoke-drift.v1", "unexpected drift schema");
assert(data.found === 2, "drift should select two full-suite runs");
assert(data.candidateArtifacts === 2, "drift runKind filter should expose eligible artifact count");
assert(data.filteredOutArtifacts === 1, "drift runKind filter should expose filtered artifact count");
assert(data.runs.every((run) => run.runKind === "full-suite"), "drift should exclude targeted run after runKind filter");
assert(data.drift.caseSetChanged === false, "filtered full-suite drift should have stable case set");
assert(data.drift.runtimeBoundsChanged === true, "drift should detect runtime bounds changes");
assert(data.drift.runtimeFingerprintChanged === true, "drift should detect runtime fingerprint changes");
assert(data.qualityTotals.recoveries === 0, "filtered full-suite drift should not include targeted recoveries");
assert(data.dimensions.runtimeBounds.length === 2, "drift should retain both runtime bounds signatures");
NODE
echo "drift JSON output did not expose expected provider/runtime drift"
cat "$drift_json"
return 1
fi
if ! output="$("$SCRIPT_PATH" --drift 3 "$tmp_dir/drift" 2>&1)"; then
echo "$output"
return 1
fi
if [[ "$output" != *"xtalpi-pi-tools smoke drift"* || "$output" != *"runtime_bounds_changed=true"* || "$output" != *"run_kind_changed=true"* || "$output" != *"quality_signals_present=true"* ]]; then
echo "drift text output did not expose expected drift indicators"
echo "$output"
return 1
fi
local retention_json="$tmp_dir/retention-output.json"
if ! output="$("$SCRIPT_PATH" --retention-report --keep-full-suite 2 --keep-targeted 0 --json "$tmp_dir/mixed-full-suite-trend" >"$retention_json" 2>&1)"; then
echo "$output"
return 1
fi
if ! node - "$retention_json" <<'NODE'; then
const fs = require("node:fs");
const data = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
function assert(condition, message) {
if (!condition) throw new Error(message);
}
assert(data.schema === "xtalpi-pi-tools.smoke-retention-report.v1", "unexpected retention schema");
assert(data.policy.action === "report_only", "retention report must be read-only");
assert(data.totals.summaryArtifacts === 4, "retention fixture should see four summary artifacts");
assert(data.runKindCounts["full-suite"] === 3, "retention should count full-suite runs");
assert(data.runKindCounts.targeted === 1, "retention should count targeted runs");
assert(data.totals.archiveCandidateRuns === 2, "retention policy should propose two archive candidates");
assert(data.archiveCandidateRunIds.join(",") === "20260702-000003,20260702-000001", "retention candidates should be newest-order policy leftovers");
assert(data.retainedRunIds.includes("20260702-000004"), "retention should keep newest full-suite");
assert(data.retainedRunIds.includes("20260702-000002"), "retention should keep second newest full-suite");
assert(data.archiveCandidateSample.every((run) => run.processLifecycleFailures === 0), "retention compact run should expose lifecycle quality fields");
assert(data.totals.runsWithoutSummary === 1, "retention should report orphan run artifacts without a summary");
assert(data.runsWithoutSummary[0].runId === "20260702-000005", "retention should expose orphan run id");
assert(data.runsWithoutSummary[0].fileCount === 2, "retention should count orphan run files");
assert(data.totals.unknownFiles === 1, "retention should report unknown files");
assert(data.unknownFiles[0].file === "manual-note.txt", "retention should expose unknown file sample");
NODE
echo "retention JSON output did not expose expected policy recommendations"
cat "$retention_json"
return 1
fi
if ! output="$("$SCRIPT_PATH" --retention-report --keep-full-suite 2 --keep-targeted 0 "$tmp_dir/mixed-full-suite-trend" 2>&1)"; then
echo "$output"
return 1
fi
if [[ "$output" != *"xtalpi-pi-tools smoke retention report"* || "$output" != *"report_only=true"* || "$output" != *"archive_candidate_runs=2"* ]]; then
echo "retention text output did not expose expected retention summary"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --history 1 --keep-full-suite 2 "$tmp_dir/mixed-full-suite-trend" 2>&1)"; then
echo "expected retention policy options without --retention-report to fail"
echo "$output"
return 1
fi
if [[ "$output" != *"--keep-* options require --retention-report"* ]]; then
echo "retention policy option misuse did not expose expected error"
echo "$output"
return 1
fi
local compare_json="$tmp_dir/compare-output.json"
if ! output="$("$SCRIPT_PATH" --compare 20260702-000002 20260702-000003 --json "$tmp_dir/history" >"$compare_json" 2>&1)"; then
echo "$output"
return 1
fi
if ! node - "$compare_json" <<'NODE'; then
const fs = require("node:fs");
const file = process.argv[2];
const data = JSON.parse(fs.readFileSync(file, "utf8"));
function assert(condition, message) {
if (!condition) throw new Error(message);
}
assert(data.schema === "xtalpi-pi-tools.smoke-compare.v1", "unexpected compare schema");
assert(data.baseRunId === "20260702-000002", "unexpected base run id");
assert(data.headRunId === "20260702-000003", "unexpected head run id");
assert(data.delta.failures === 1, "failure delta should show regression");
assert(data.delta.rawToolMarkupFinalAnswers === 1, "raw markup delta should show regression");
assert(data.delta.recoveries === -1, "recovery delta should be head-base");
assert(data.okChanged === true, "ok change should be visible");
assert(data.caseSetChanged === false, "case set should be stable for read-only history fixtures");
assert(data.caseDeltas.some((item) => item.caseName === "read" && item.delta.rawToolMarkupFinalAnswer === true), "case-level raw markup change missing");
NODE
return 1
fi
if ! output="$("$SCRIPT_PATH" --compare 20260702-000002 20260702-000003 "$tmp_dir/history" 2>&1)"; then
echo "$output"
return 1
fi
if [[ "$output" != *"xtalpi-pi-tools smoke compare"* || "$output" != *"failures_delta=1"* || "$output" != *"raw_tool_markup_final_answers_delta=1"* || "$output" != *"case_set_changed=false"* ]]; then
echo "compare text output did not expose expected regression deltas"
echo "$output"
return 1
fi
local trend_json="$tmp_dir/trend-output.json"
if ! output="$("$SCRIPT_PATH" --trend-gate 2 --json "$tmp_dir/trend" >"$trend_json" 2>&1)"; then
echo "$output"
return 1
fi
if ! node - "$trend_json" <<'NODE'; then
const fs = require("node:fs");
const file = process.argv[2];
const data = JSON.parse(fs.readFileSync(file, "utf8"));
function assert(condition, message) {
if (!condition) throw new Error(message);
}
assert(data.schema === "xtalpi-pi-tools.smoke-trend-gate.v1", "unexpected trend-gate schema");
assert(data.gateFailures.length === 0, "clean trend gate should pass");
assert(data.history.runs.length === 2, "trend gate should inspect two runs");
assert(data.recoveryTrend.recoveryDelta === 1, "recovery trend should preserve the latest delta");
NODE
return 1
fi
if ! output="$("$SCRIPT_PATH" --trend-gate 2 --expect-cases 5 --expect-case-names no-tool,bash,read,bash-read,web-read "$tmp_dir/trend" 2>&1)"; then
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 --require-run-kind full-suite "$tmp_dir/trend" 2>&1)"; then
echo "expected require-run-kind to fail targeted trend fixture"
echo "$output"
return 1
fi
if [[ "$output" != *"expected run_kind=full-suite"* ]]; then
echo "require-run-kind failure did not expose runKind mismatch"
echo "$output"
return 1
fi
local profile_json="$tmp_dir/profile-output.json"
if ! output="$("$SCRIPT_PATH" --trend-gate 2 --profile full-suite-strict --json "$tmp_dir/full-suite-trend" >"$profile_json" 2>&1)"; then
echo "$output"
return 1
fi
if ! node - "$profile_json" <<'NODE'; then
const fs = require("node:fs");
const file = process.argv[2];
const data = JSON.parse(fs.readFileSync(file, "utf8"));
function assert(condition, message) {
if (!condition) throw new Error(message);
}
assert(data.ok === true, "full-suite-strict profile should pass for clean full-suite history");
assert(data.limits.profile === "full-suite-strict", "profile should be visible in trend gate limits");
assert(data.limits.expectCases === 12, "profile should set full-suite case count");
assert(data.limits.maxRecoveries === 2, "profile should allow bounded local repair");
assert(data.limits.maxRecoveryRate === 0.15, "profile should cap local repair rate");
assert(data.limits.maxRecoveryCaseRuns === 3, "profile should cap repeated recovered case runs");
assert(data.limits.failOnRecoveryIncrease !== true, "profile should not fail solely on bounded repair increase");
assert(data.history.runs.every((run) => run.runKind === "full-suite"), "profile fixture should classify full-suite runs");
assert(
JSON.stringify(data.limits.expectCaseNames) === JSON.stringify([
"bash",
"bash-read",
"no-tool",
"plan-mode-accepted-continuation",
"plan-mode-contract",
"read",
"read-enoent-recovery",
"tool-result-injection",
"tool-selection-clipping",
"tool-selection-continuation",
"until-done-continuation",
"web-read",
]),
"profile should set exact full-suite case names",
);
NODE
return 1
fi
local ranking_profile_json="$tmp_dir/ranking-profile-output.json"
if ! output="$("$SCRIPT_PATH" --trend-gate 2 --profile full-suite-ranking-strict --json "$tmp_dir/full-suite-trend" >"$ranking_profile_json" 2>&1)"; then
echo "$output"
return 1
fi
if ! node - "$ranking_profile_json" <<'NODE'; then
const fs = require("node:fs");
const file = process.argv[2];
const data = JSON.parse(fs.readFileSync(file, "utf8"));
function assert(condition, message) {
if (!condition) throw new Error(message);
}
assert(data.ok === true, "full-suite-ranking-strict profile should pass clean full-suite history");
assert(data.limits.profile === "full-suite-ranking-strict", "ranking profile should be visible in trend gate limits");
assert(JSON.stringify(data.limits.requireToolSelectionReasonCodes) === JSON.stringify(["core_tool", "prompt_path_file"]), "ranking profile should require aggregate reason codes");
assert(JSON.stringify(data.limits.requireSelectedToolSelectionReasonCodes) === JSON.stringify(["core_tool", "prompt_path_file"]), "ranking profile should require selected-tool reason codes");
assert(JSON.stringify(data.limits.requireOmittedToolSelectionReasonCodes) === JSON.stringify(["core_tool"]), "ranking profile should require omitted-tool reason codes");
assert(JSON.stringify(data.limits.forbidToolSelectionReasonCodes) === JSON.stringify(["prompt_tool_exclusive"]), "ranking profile should forbid explicit-only leakage in full suite");
assert(data.limits.requireStableRuntimeFingerprint !== true, "ranking profile should not require runtime fingerprint stability");
assert(data.limits.requireStableRuntimeBounds !== true, "ranking profile should not require runtime bounds stability");
assert(data.history.runs.every((run) => run.runKind === "full-suite"), "ranking profile fixture should classify full-suite runs");
NODE
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 --profile full-suite-ranking-strict "$tmp_dir/ranking-drift-trend" 2>&1)"; then
echo "expected full-suite-ranking-strict profile to fail reason-code drift fixture"
echo "$output"
return 1
fi
if [[ "$output" != *"tool_selection_reason_codes"* || "$output" != *"prompt_tool_exclusive"* || "$output" != *"selected_tool_selection_reason_codes"* || "$output" != *"prompt_path_file"* || "$output" != *"omitted_tool_selection_reason_codes"* || "$output" != *"core_tool"* ]]; then
echo "ranking strict profile failure did not expose reason-code drift"
echo "$output"
return 1
fi
local runtime_profile_json="$tmp_dir/runtime-profile-output.json"
if ! output="$("$SCRIPT_PATH" --trend-gate 2 --profile full-suite-runtime-strict --json "$tmp_dir/full-suite-trend" >"$runtime_profile_json" 2>&1)"; then
echo "$output"
return 1
fi
if ! node - "$runtime_profile_json" <<'NODE'; then
const fs = require("node:fs");
const file = process.argv[2];
const data = JSON.parse(fs.readFileSync(file, "utf8"));
function assert(condition, message) {
if (!condition) throw new Error(message);
}
assert(data.ok === true, "full-suite-runtime-strict profile should pass stable full-suite history");
assert(data.limits.profile === "full-suite-runtime-strict", "runtime profile should be visible in limits");
assert(data.limits.requireStableRuntimeFingerprint === true, "runtime profile should require stable fingerprint");
assert(data.limits.requireStableRuntimeBounds === true, "runtime profile should require stable bounds");
assert(data.runtimeStability.runtimeFingerprints.length === 1, "stable fixture should have one runtime fingerprint");
assert(data.runtimeStability.runtimeBounds.length === 1, "stable fixture should have one runtime bounds signature");
NODE
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 --profile full-suite-runtime-strict "$tmp_dir/drift" 2>&1)"; then
echo "expected full-suite-runtime-strict profile to fail runtime drift fixture"
echo "$output"
return 1
fi
if [[ "$output" != *"runtime_fingerprint changed across selected runs"* || "$output" != *"runtime_bounds changed across selected runs"* ]]; then
echo "runtime strict profile failure did not expose runtime drift reasons"
echo "$output"
return 1
fi
local mixed_profile_json="$tmp_dir/mixed-profile-output.json"
if ! output="$("$SCRIPT_PATH" --trend-gate 3 --profile full-suite-strict --json "$tmp_dir/mixed-full-suite-trend" >"$mixed_profile_json" 2>&1)"; then
echo "$output"
return 1
fi
if ! node - "$mixed_profile_json" <<'NODE'; then
const fs = require("node:fs");
const file = process.argv[2];
const data = JSON.parse(fs.readFileSync(file, "utf8"));
function assert(condition, message) {
if (!condition) throw new Error(message);
}
assert(data.ok === true, "profile should pass newest full-suite runs when a targeted run is interleaved");
assert(data.history.totalArtifacts === 4, "mixed fixture should preserve total artifact count");
assert(data.history.candidateArtifacts === 3, "profile should see three eligible full-suite artifacts");
assert(data.history.filteredOutArtifacts === 1, "profile should filter out one targeted artifact");
assert(JSON.stringify(data.history.filter.runKinds) === JSON.stringify(["full-suite"]), "profile should filter runKind=full-suite");
assert(data.history.found === 3, "profile should select three full-suite runs");
assert(data.history.runs.map((run) => run.runId).join(",") === "20260702-000004,20260702-000002,20260702-000001", "profile selected wrong full-suite runs");
assert(data.history.runs.every((run) => run.runKind === "full-suite"), "profile selected a non-full-suite run");
NODE
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 --profile full-suite-strict "$tmp_dir/trend" 2>&1)"; then
echo "expected full-suite-strict profile to fail non-full-suite trend fixture"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 --expect-case-names no-tool,bash,read,bash-read,tool-result-injection "$tmp_dir/trend" 2>&1)"; then
echo "expected trend case-name gate to fail"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 --expect-cases 5 "$tmp_dir/subset-trend" 2>&1)"; then
echo "expected subset trend fixture to fail case-count gate"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 "$tmp_dir/short-trend" 2>&1)"; then
echo "expected short trend fixture to fail insufficient-history gate"
echo "$output"
return 1
fi
if [[ "$output" != *"expected at least 2 summary artifacts, found 1"* ]]; then
echo "insufficient-history trend gate did not expose the requested/found mismatch"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 --max-recoveries 0 "$tmp_dir/trend" 2>&1)"; then
echo "expected recovery threshold trend gate to fail"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 --fail-on-recovery-increase "$tmp_dir/trend" 2>&1)"; then
echo "expected recovery increase trend gate to fail"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 --max-request-latency-ms 60000 "$tmp_dir/latency-trend" 2>&1)"; then
echo "expected latency threshold trend gate to fail"
echo "$output"
return 1
fi
if [[ "$output" != *"expected request_latency_ms_max<=60000, got 62000"* ]]; then
echo "latency threshold trend gate did not expose expected failure"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 --max-slow-requests 0 "$tmp_dir/latency-trend" 2>&1)"; then
echo "expected slow request trend gate to fail"
echo "$output"
return 1
fi
if [[ "$output" != *"expected slow_requests<=0, got 1"* ]]; then
echo "slow request trend gate did not expose expected failure"
echo "$output"
return 1
fi
if output="$("$SCRIPT_PATH" --trend-gate 2 "$tmp_dir/history" 2>&1)"; then
echo "expected failed/raw history trend gate to fail"
echo "$output"
return 1
fi
echo "xtalpi-pi-tools debug summary self-test passed"
}
if [ "${1:-}" = "--self-test" ]; then
run_self_test
exit 0
fi
while [ "$#" -gt 0 ]; do
case "$1" in
--json)
FORMAT="json"
shift
;;
--profile)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --profile requires NAME" >&2
exit 2
fi
PROFILE="${2:-}"
shift 2
;;
--latest)
LATEST_ONLY="1"
shift
;;
--run-id)
RUN_ID="${2:-}"
shift 2
;;
--history)
HISTORY_LIMIT="${2:-}"
shift 2
;;
--trend-gate)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --trend-gate requires N" >&2
exit 2
fi
TREND_GATE_LIMIT="${2:-}"
shift 2
;;
--drift)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --drift requires N" >&2
exit 2
fi
DRIFT_LIMIT="${2:-}"
shift 2
;;
--retention-report)
RETENTION_REPORT="1"
shift
;;
--compare)
if [ "$#" -lt 3 ]; then
echo "xtalpi-pi-tools debug summary: --compare requires BASE_RUN and HEAD_RUN" >&2
exit 2
fi
COMPARE_BASE_RUN_ID="${2:-}"
COMPARE_HEAD_RUN_ID="${3:-}"
shift 3
;;
--expect-cases)
EXPECT_CASES="${2:-}"
shift 2
;;
--expect-case-names|--expect-selected-cases)
EXPECT_CASE_NAMES="${2:-}"
shift 2
;;
--run-kind|--filter-run-kind)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --run-kind requires LIST" >&2
exit 2
fi
RUN_KIND_FILTER="${2:-}"
shift 2
;;
--require-run-kind)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --require-run-kind requires LIST" >&2
exit 2
fi
REQUIRE_RUN_KIND="${2:-}"
shift 2
;;
--max-errors)
MAX_ERRORS="${2:-}"
shift 2
;;
--max-empty-assistant-ends)
MAX_EMPTY_ASSISTANT_ENDS="${2:-}"
shift 2
;;
--max-raw-tool-markup-final-answers|--max-tool-envelope-final-answers)
MAX_RAW_TOOL_MARKUP_FINAL_ANSWERS="${2:-}"
shift 2
;;
--max-recoveries)
MAX_RECOVERIES="${2:-}"
shift 2
;;
--max-recovery-rate)
MAX_RECOVERY_RATE="${2:-}"
shift 2
;;
--max-request-latency-ms)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --max-request-latency-ms requires N" >&2
exit 2
fi
MAX_REQUEST_LATENCY_MS="${2:-}"
shift 2
;;
--max-slow-requests)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --max-slow-requests requires N" >&2
exit 2
fi
MAX_SLOW_REQUESTS="${2:-}"
shift 2
;;
--require-tool-selection-reason-codes)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --require-tool-selection-reason-codes requires LIST" >&2
exit 2
fi
REQUIRE_TOOL_SELECTION_REASON_CODES="${2:-}"
shift 2
;;
--require-selected-tool-selection-reason-codes)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --require-selected-tool-selection-reason-codes requires LIST" >&2
exit 2
fi
REQUIRE_SELECTED_TOOL_SELECTION_REASON_CODES="${2:-}"
shift 2
;;
--require-omitted-tool-selection-reason-codes)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --require-omitted-tool-selection-reason-codes requires LIST" >&2
exit 2
fi
REQUIRE_OMITTED_TOOL_SELECTION_REASON_CODES="${2:-}"
shift 2
;;
--forbid-tool-selection-reason-codes)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --forbid-tool-selection-reason-codes requires LIST" >&2
exit 2
fi
FORBID_TOOL_SELECTION_REASON_CODES="${2:-}"
shift 2
;;
--forbid-selected-tool-selection-reason-codes)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --forbid-selected-tool-selection-reason-codes requires LIST" >&2
exit 2
fi
FORBID_SELECTED_TOOL_SELECTION_REASON_CODES="${2:-}"
shift 2
;;
--forbid-omitted-tool-selection-reason-codes)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --forbid-omitted-tool-selection-reason-codes requires LIST" >&2
exit 2
fi
FORBID_OMITTED_TOOL_SELECTION_REASON_CODES="${2:-}"
shift 2
;;
--fail-on-recovery-increase)
FAIL_ON_RECOVERY_INCREASE="1"
shift
;;
--max-recovery-case-runs)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --max-recovery-case-runs requires N" >&2
exit 2
fi
MAX_RECOVERY_CASE_RUNS="${2:-}"
shift 2
;;
--require-stable-runtime)
REQUIRE_STABLE_RUNTIME_FINGERPRINT="1"
REQUIRE_STABLE_RUNTIME_BOUNDS="1"
shift
;;
--require-stable-runtime-fingerprint)
REQUIRE_STABLE_RUNTIME_FINGERPRINT="1"
shift
;;
--require-stable-runtime-bounds)
REQUIRE_STABLE_RUNTIME_BOUNDS="1"
shift
;;
--keep-full-suite)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --keep-full-suite requires N" >&2
exit 2
fi
RETENTION_POLICY_OPTION_USED="1"
KEEP_FULL_SUITE="${2:-}"
shift 2
;;
--keep-targeted)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --keep-targeted requires N" >&2
exit 2
fi
RETENTION_POLICY_OPTION_USED="1"
KEEP_TARGETED="${2:-}"
shift 2
;;
--keep-preflight-failed)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --keep-preflight-failed requires N" >&2
exit 2
fi
RETENTION_POLICY_OPTION_USED="1"
KEEP_PREFLIGHT_FAILED="${2:-}"
shift 2
;;
--keep-empty)
if [ "$#" -lt 2 ]; then
echo "xtalpi-pi-tools debug summary: --keep-empty requires N" >&2
exit 2
fi
RETENTION_POLICY_OPTION_USED="1"
KEEP_EMPTY="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
OUT_DIR="$1"
shift
;;
esac
done
apply_profile_defaults
node - \
"$SUMMARY_CORE_PATH" \
"$OUT_DIR" \
"$FORMAT" \
"$PROFILE" \
"$LATEST_ONLY" \
"$RUN_ID" \
"$HISTORY_LIMIT" \
"$TREND_GATE_LIMIT" \
"$DRIFT_LIMIT" \
"$COMPARE_BASE_RUN_ID" \
"$COMPARE_HEAD_RUN_ID" \
"$FAIL_ON_RECOVERY_INCREASE" \
"$MAX_RECOVERY_CASE_RUNS" \
"$EXPECT_CASES" \
"$EXPECT_CASE_NAMES" \
"$MAX_ERRORS" \
"$MAX_EMPTY_ASSISTANT_ENDS" \
"$MAX_RAW_TOOL_MARKUP_FINAL_ANSWERS" \
"$MAX_RECOVERIES" \
"$MAX_RECOVERY_RATE" \
"$MAX_REQUEST_LATENCY_MS" \
"$MAX_SLOW_REQUESTS" \
"$REQUIRE_TOOL_SELECTION_REASON_CODES" \
"$REQUIRE_SELECTED_TOOL_SELECTION_REASON_CODES" \
"$REQUIRE_OMITTED_TOOL_SELECTION_REASON_CODES" \
"$FORBID_TOOL_SELECTION_REASON_CODES" \
"$FORBID_SELECTED_TOOL_SELECTION_REASON_CODES" \
"$FORBID_OMITTED_TOOL_SELECTION_REASON_CODES" \
"$RUN_KIND_FILTER" \
"$REQUIRE_RUN_KIND" \
"$REQUIRE_STABLE_RUNTIME_FINGERPRINT" \
"$REQUIRE_STABLE_RUNTIME_BOUNDS" \
"$RETENTION_REPORT" \
"$RETENTION_POLICY_OPTION_USED" \
"$KEEP_FULL_SUITE" \
"$KEEP_TARGETED" \
"$KEEP_PREFLIGHT_FAILED" \
"$KEEP_EMPTY" <<'NODE'
const fs = require("node:fs");
const crypto = require("node:crypto");
const path = require("node:path");
const [
summaryCorePath,
outDir,
format,
gateProfileRaw,
latestOnlyRaw,
runIdRaw,
historyLimitRaw,
trendGateLimitRaw,
driftLimitRaw,
compareBaseRunIdRaw,
compareHeadRunIdRaw,
failOnRecoveryIncreaseRaw,
maxRecoveryCaseRunsRaw,
expectCasesRaw,
expectCaseNamesRaw,
maxErrorsRaw,
maxEmptyAssistantEndsRaw,
maxRawToolMarkupFinalAnswersRaw,
maxRecoveriesRaw,
maxRecoveryRateRaw,
maxRequestLatencyMsRaw,
maxSlowRequestsRaw,
requireToolSelectionReasonCodesRaw,
requireSelectedToolSelectionReasonCodesRaw,
requireOmittedToolSelectionReasonCodesRaw,
forbidToolSelectionReasonCodesRaw,
forbidSelectedToolSelectionReasonCodesRaw,
forbidOmittedToolSelectionReasonCodesRaw,
runKindFilterRaw,
requireRunKindRaw,
requireStableRuntimeFingerprintRaw,
requireStableRuntimeBoundsRaw,
retentionReportRaw,
retentionPolicyOptionUsedRaw,
keepFullSuiteRaw,
keepTargetedRaw,
keepPreflightFailedRaw,
keepEmptyRaw,
] = process.argv.slice(2);
const {
buildCaseSet,
classifyRunKind,
containsRawPiToolMarkup,
isRawToolMarkupFinalAnswer,
isToolEnvelopeOnlyFinalAnswer,
normalizeCaseSet,
numberOrUndefined,
numberOrZero,
objectOrUndefined,
readJsonFile,
readJsonl,
sortedUniqueStrings,
stripPiToolEnvelopes,
uniqueBooleans,
uniqueNumbers,
uniqueStrings,
} = require(summaryCorePath);
const latestOnly = latestOnlyRaw === "1";
const gateProfile = String(gateProfileRaw || "").trim();
const runIdFilter = String(runIdRaw || "").trim();
const failOnRecoveryIncrease = failOnRecoveryIncreaseRaw === "1";
const compareBaseRunId = String(compareBaseRunIdRaw || "").trim();
const compareHeadRunId = String(compareHeadRunIdRaw || "").trim();
function optionalNumber(raw, name) {
if (raw === undefined || raw === "") return undefined;
const value = Number(raw);
if (!Number.isFinite(value) || value < 0) {
console.error(`xtalpi-pi-tools debug summary: ${name} must be a non-negative number`);
process.exit(2);
}
return value;
}
function requiredNonNegativeInteger(raw, name) {
const value = optionalNumber(raw, name);
if (value === undefined || !Number.isInteger(value)) {
console.error(`xtalpi-pi-tools debug summary: ${name} must be a non-negative integer`);
process.exit(2);
}
return value;
}
function parseCaseNameList(raw, name) {
if (raw === undefined || raw === "") return undefined;
const list = String(raw).split(",").map((item) => item.trim());
if (list.length === 0 || list.some((item) => item === "")) {
console.error(`xtalpi-pi-tools debug summary: ${name} must be a comma-separated non-empty case list`);
process.exit(2);
}
return sortedUniqueStrings(list);
}
function parseRunKindList(raw, name) {
if (raw === undefined || raw === "") return undefined;
const list = String(raw).split(",").map((item) => item.trim());
if (list.length === 0 || list.some((item) => item === "")) {
console.error(`xtalpi-pi-tools debug summary: ${name} must be a comma-separated non-empty runKind list`);
process.exit(2);
}
return sortedUniqueStrings(list);
}
function parseReasonCodeList(raw, name) {
if (raw === undefined || raw === "") return undefined;
const list = String(raw).split(",").map((item) => item.trim());
if (list.length === 0 || list.some((item) => item === "")) {
console.error(`xtalpi-pi-tools debug summary: ${name} must be a comma-separated non-empty reason-code list`);
process.exit(2);
}
return sortedUniqueStrings(list);
}
function caseNameListsEqual(actual, expected) {
const actualSorted = sortedUniqueStrings(actual);
const expectedSorted = sortedUniqueStrings(expected);
return actualSorted.length === expectedSorted.length &&
actualSorted.every((item, index) => item === expectedSorted[index]);
}
function runKindMatches(runKind, expectedKinds) {
if (expectedKinds === undefined) return true;
return expectedKinds.includes(String(runKind || ""));
}
function formatRunKinds(values) {
return sortedUniqueStrings(values).join(",");
}
function formatCaseNames(values) {
return sortedUniqueStrings(values).join(",");
}
function stableValue(value) {
if (Array.isArray(value)) return value.map(stableValue);
if (value && typeof value === "object") {
return Object.fromEntries(
Object.entries(value)
.filter(([, child]) => child !== undefined)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, child]) => [key, stableValue(child)]),
);
}
return value;
}
function stableStringify(value) {
return JSON.stringify(stableValue(value));
}
function sha256Hex(value) {
return crypto.createHash("sha256").update(stableStringify(value)).digest("hex");
}
function shortSha256(value) {
return sha256Hex(value).slice(0, 16);
}
function boolOrUndefinedLocal(value) {
return typeof value === "boolean" ? value : undefined;
}
function compactObject(value) {
return Object.fromEntries(Object.entries(value).filter(([, child]) => child !== undefined));
}
const SLOW_REQUEST_THRESHOLD_MS = 60000;
const runKindFilter = parseRunKindList(runKindFilterRaw, "--run-kind");
const requireRunKinds = parseRunKindList(requireRunKindRaw, "--require-run-kind");
const requireStableRuntimeFingerprint = requireStableRuntimeFingerprintRaw === "1";
const requireStableRuntimeBounds = requireStableRuntimeBoundsRaw === "1";
const retentionReport = retentionReportRaw === "1";
const retentionPolicyOptionUsed = retentionPolicyOptionUsedRaw === "1";
const retentionPolicy = {
action: "report_only",
keepFullSuite: requiredNonNegativeInteger(keepFullSuiteRaw, "--keep-full-suite"),
keepTargeted: requiredNonNegativeInteger(keepTargetedRaw, "--keep-targeted"),
keepPreflightFailed: requiredNonNegativeInteger(keepPreflightFailedRaw, "--keep-preflight-failed"),
keepEmpty: requiredNonNegativeInteger(keepEmptyRaw, "--keep-empty"),
keepQualitySignalRuns: true,
sampleLimit: 20,
};
const gates = {
profile: gateProfile || undefined,
expectCases: optionalNumber(expectCasesRaw, "--expect-cases"),
expectCaseNames: parseCaseNameList(expectCaseNamesRaw, "--expect-case-names"),
runKindFilter,
requireRunKinds,
maxErrors: optionalNumber(maxErrorsRaw, "--max-errors"),
maxEmptyAssistantEnds: optionalNumber(maxEmptyAssistantEndsRaw, "--max-empty-assistant-ends"),
maxRawToolMarkupFinalAnswers: optionalNumber(
maxRawToolMarkupFinalAnswersRaw,
"--max-raw-tool-markup-final-answers",
),
maxRecoveries: optionalNumber(maxRecoveriesRaw, "--max-recoveries"),
maxRecoveryRate: optionalNumber(maxRecoveryRateRaw, "--max-recovery-rate"),
maxRequestLatencyMs: optionalNumber(maxRequestLatencyMsRaw, "--max-request-latency-ms"),
maxSlowRequests: optionalNumber(maxSlowRequestsRaw, "--max-slow-requests"),
requireToolSelectionReasonCodes: parseReasonCodeList(
requireToolSelectionReasonCodesRaw,
"--require-tool-selection-reason-codes",
),
requireSelectedToolSelectionReasonCodes: parseReasonCodeList(
requireSelectedToolSelectionReasonCodesRaw,
"--require-selected-tool-selection-reason-codes",
),
requireOmittedToolSelectionReasonCodes: parseReasonCodeList(
requireOmittedToolSelectionReasonCodesRaw,
"--require-omitted-tool-selection-reason-codes",
),
forbidToolSelectionReasonCodes: parseReasonCodeList(
forbidToolSelectionReasonCodesRaw,
"--forbid-tool-selection-reason-codes",
),
forbidSelectedToolSelectionReasonCodes: parseReasonCodeList(
forbidSelectedToolSelectionReasonCodesRaw,
"--forbid-selected-tool-selection-reason-codes",
),
forbidOmittedToolSelectionReasonCodes: parseReasonCodeList(
forbidOmittedToolSelectionReasonCodesRaw,
"--forbid-omitted-tool-selection-reason-codes",
),
requireStableRuntimeFingerprint,
requireStableRuntimeBounds,
};
const historyLimit = optionalNumber(historyLimitRaw, "--history");
if (historyLimit !== undefined && (!Number.isInteger(historyLimit) || historyLimit < 1)) {
console.error("xtalpi-pi-tools debug summary: --history must be a positive integer");
process.exit(2);
}
const trendGateLimit = optionalNumber(trendGateLimitRaw, "--trend-gate");
if (trendGateLimit !== undefined && (!Number.isInteger(trendGateLimit) || trendGateLimit < 1)) {
console.error("xtalpi-pi-tools debug summary: --trend-gate must be a positive integer");
process.exit(2);
}
const driftLimit = optionalNumber(driftLimitRaw, "--drift");
if (driftLimit !== undefined && (!Number.isInteger(driftLimit) || driftLimit < 1)) {
console.error("xtalpi-pi-tools debug summary: --drift must be a positive integer");
process.exit(2);
}
const maxRecoveryCaseRuns = optionalNumber(maxRecoveryCaseRunsRaw, "--max-recovery-case-runs");
if (
maxRecoveryCaseRuns !== undefined &&
(!Number.isInteger(maxRecoveryCaseRuns) || maxRecoveryCaseRuns < 0)
) {
console.error("xtalpi-pi-tools debug summary: --max-recovery-case-runs must be a non-negative integer");
process.exit(2);
}
if (gates.maxSlowRequests !== undefined && !Number.isInteger(gates.maxSlowRequests)) {
console.error("xtalpi-pi-tools debug summary: --max-slow-requests must be a non-negative integer");
process.exit(2);
}
const compareMode = compareBaseRunId !== "" || compareHeadRunId !== "";
if (compareMode && (compareBaseRunId === "" || compareHeadRunId === "")) {
console.error("xtalpi-pi-tools debug summary: --compare requires BASE_RUN and HEAD_RUN");
process.exit(2);
}
if (compareMode && (historyLimit !== undefined || trendGateLimit !== undefined || driftLimit !== undefined || retentionReport)) {
console.error("xtalpi-pi-tools debug summary: --compare cannot be combined with --history, --trend-gate, --drift, or --retention-report");
process.exit(2);
}
if (compareMode && (runKindFilter !== undefined || requireRunKinds !== undefined)) {
console.error("xtalpi-pi-tools debug summary: --compare cannot be combined with --run-kind or --require-run-kind");
process.exit(2);
}
if (compareMode && (latestOnly || runIdFilter !== "")) {
console.error("xtalpi-pi-tools debug summary: --compare cannot be combined with --latest or --run-id");
process.exit(2);
}
const aggregateModeCount = [historyLimit, trendGateLimit, driftLimit, retentionReport ? 1 : undefined]
.filter((value) => value !== undefined).length;
if (aggregateModeCount > 1) {
console.error("xtalpi-pi-tools debug summary: --history, --trend-gate, --drift, and --retention-report are mutually exclusive");
process.exit(2);
}
if (!retentionReport && retentionPolicyOptionUsed) {
console.error("xtalpi-pi-tools debug summary: --keep-* options require --retention-report");
process.exit(2);
}
if ((trendGateLimit !== undefined || driftLimit !== undefined) && (latestOnly || runIdFilter !== "")) {
console.error("xtalpi-pi-tools debug summary: --trend-gate/--drift cannot be combined with --latest or --run-id");
process.exit(2);
}
if (driftLimit !== undefined && requireRunKinds !== undefined) {
console.error("xtalpi-pi-tools debug summary: --require-run-kind cannot be combined with --drift; use --run-kind to select drift candidates");
process.exit(2);
}
if ((requireStableRuntimeFingerprint || requireStableRuntimeBounds) && trendGateLimit === undefined) {
console.error("xtalpi-pi-tools debug summary: --require-stable-runtime* requires --trend-gate");
process.exit(2);
}
if (retentionReport && (latestOnly || runIdFilter !== "")) {
console.error("xtalpi-pi-tools debug summary: --retention-report cannot be combined with --latest or --run-id");
process.exit(2);
}
if (retentionReport && (runKindFilter !== undefined || requireRunKinds !== undefined)) {
console.error("xtalpi-pi-tools debug summary: --retention-report cannot be combined with --run-kind or --require-run-kind");
process.exit(2);
}
if (retentionReport && (
gateProfile ||
failOnRecoveryIncrease ||
maxRecoveryCaseRunsRaw !== "" ||
expectCasesRaw !== "" ||
expectCaseNamesRaw !== "" ||
maxErrorsRaw !== "0" ||
maxEmptyAssistantEndsRaw !== "" ||
maxRawToolMarkupFinalAnswersRaw !== "" ||
maxRecoveriesRaw !== "" ||
maxRecoveryRateRaw !== "" ||
maxRequestLatencyMsRaw !== "" ||
maxSlowRequestsRaw !== "" ||
requireToolSelectionReasonCodesRaw !== "" ||
requireSelectedToolSelectionReasonCodesRaw !== "" ||
requireOmittedToolSelectionReasonCodesRaw !== "" ||
forbidToolSelectionReasonCodesRaw !== "" ||
forbidSelectedToolSelectionReasonCodesRaw !== "" ||
forbidOmittedToolSelectionReasonCodesRaw !== "" ||
requireStableRuntimeFingerprint ||
requireStableRuntimeBounds
)) {
console.error("xtalpi-pi-tools debug summary: --retention-report cannot be combined with trend gate options");
process.exit(2);
}
if (historyLimit !== undefined && (latestOnly || runIdFilter !== "")) {
console.error("xtalpi-pi-tools debug summary: --history cannot be combined with --latest or --run-id");
process.exit(2);
}
if (runKindFilter !== undefined && historyLimit === undefined && trendGateLimit === undefined && driftLimit === undefined) {
console.error("xtalpi-pi-tools debug summary: --run-kind requires --history, --trend-gate, or --drift");
process.exit(2);
}
function increment(map, key, by = 1) {
map[key] = (map[key] ?? 0) + by;
}
function inferCaseParts(fileName) {
const stem = fileName.replace(/\.debug\.jsonl$/, "");
const match = stem.match(/^(\d{8}-\d{6}(?:-\d+)?)-(.+)$/);
return match ? { runId: match[1], caseName: match[2] } : { runId: "unknown", caseName: stem };
}
function finalAssistantText(events) {
const agentEvents = events.filter((event) => event.type === "agent_end");
const agent = agentEvents.at(-1);
const final = agent?.messages?.filter((message) => message.role === "assistant").at(-1);
if (!Array.isArray(final?.content)) return "";
return final.content
.filter((block) => block.type === "text")
.map((block) => block.text)
.join("\n");
}
function eventData(event) {
return objectOrUndefined(event?.data) || {};
}
function stringMetric(event, camelName, snakeName) {
const direct = event?.[snakeName];
if (typeof direct === "string") return direct;
const fromData = eventData(event)[camelName];
return typeof fromData === "string" ? fromData : undefined;
}
function stringArrayMetric(event, camelName, snakeName) {
const normalize = (value) => {
if (Array.isArray(value)) return value.map(String).filter(Boolean);
if (typeof value === "string") return value.split(",").map((item) => item.trim()).filter(Boolean);
return [];
};
const direct = normalize(event?.[snakeName]);
if (direct.length > 0) return direct;
return normalize(eventData(event)[camelName]);
}
function numberMetric(event, camelName, snakeName) {
const direct = numberOrUndefined(event?.[snakeName]);
if (direct !== undefined) return direct;
return numberOrUndefined(eventData(event)[camelName]);
}
function booleanMetric(event, camelName, snakeName) {
const direct = event?.[snakeName];
if (typeof direct === "boolean") return direct;
const fromData = eventData(event)[camelName];
return typeof fromData === "boolean" ? fromData : undefined;
}
function argumentValidationWarnings(event) {
const data = eventData(event);
return Array.isArray(data.argumentValidationWarnings) ? data.argumentValidationWarnings : [];
}
function argumentValidationWarningCount(event) {
const direct = numberMetric(
event,
"argumentValidationWarningCount",
"argument_validation_warning_count",
);
if (direct !== undefined) return direct;
return argumentValidationWarnings(event).length;
}
function argumentValidationWarningCodes(event) {
return uniqueStrings([
...stringArrayMetric(event, "argumentValidationWarningCodes", "argument_validation_warning_codes"),
...argumentValidationWarnings(event).map((warning) => warning?.code),
]);
}
function reasonCodesFromValue(value) {
if (Array.isArray(value)) {
return uniqueStrings(value.map((item) => (item === undefined || item === null ? "" : String(item).trim())));
}
if (typeof value === "string") {
return uniqueStrings(value.split(",").map((item) => item.trim()));
}
return [];
}
function incrementReasonCodes(target, codes) {
for (const code of reasonCodesFromValue(codes)) {
increment(target, code);
}
}
function collectToolSelectionReasonCodeCounts(event) {
const selected = {};
const omitted = {};
const all = {};
const append = (target, codes) => {
const normalizedCodes = reasonCodesFromValue(codes);
incrementReasonCodes(target, normalizedCodes);
incrementReasonCodes(all, normalizedCodes);
};
const summary = [
objectOrUndefined(eventData(event).toolSelectionSummary),
objectOrUndefined(event?.toolSelectionSummary),
objectOrUndefined(event?.tool_selection_summary),
].find(Boolean);
let sawSelectedItems = false;
let sawOmittedItems = false;
if (summary) {
const selectedItems = Array.isArray(summary.selected) ? summary.selected : [];
const omittedItems = Array.isArray(summary.omitted) ? summary.omitted : [];
if (selectedItems.length > 0) sawSelectedItems = true;
if (omittedItems.length > 0) sawOmittedItems = true;
for (const item of selectedItems) append(selected, item?.reasonCodes);
for (const item of omittedItems) append(omitted, item?.reasonCodes);
}
if (!sawSelectedItems) {
append(selected, stringArrayMetric(event, "selectedToolSelectionReasonCodes", "selected_tool_selection_reason_codes"));
}
if (!sawOmittedItems) {
append(omitted, stringArrayMetric(event, "omittedToolSelectionReasonCodes", "omitted_tool_selection_reason_codes"));
}
if (Object.keys(all).length === 0) {
incrementReasonCodes(all, stringArrayMetric(event, "toolSelectionReasonCodes", "tool_selection_reason_codes"));
}
return {
toolSelectionReasonCodes: all,
selectedToolSelectionReasonCodes: selected,
omittedToolSelectionReasonCodes: omitted,
};
}
function eventTimestampMs(event) {
if (typeof event?.ts !== "string" || event.ts.trim() === "") return undefined;
const value = Date.parse(event.ts);
return Number.isFinite(value) ? value : undefined;
}
function collectRequestLatencySummary(events) {
const latencies = [];
for (let index = 0; index < events.length; index += 1) {
const event = events[index];
if (event?.event !== "request") continue;
const requestTs = eventTimestampMs(event);
if (requestTs === undefined) continue;
for (let cursor = index + 1; cursor < events.length; cursor += 1) {
const terminal = events[cursor];
if (terminal?.event !== "response" && terminal?.event !== "error.provider") continue;
const terminalTs = eventTimestampMs(terminal);
if (terminalTs !== undefined && terminalTs >= requestTs) {
latencies.push(terminalTs - requestTs);
}
break;
}
}
const requestCount = latencies.length;
const requestLatencyMsTotal = latencies.reduce((sum, value) => sum + value, 0);
return {
requestCount,
requestLatencyMsMin: requestCount > 0 ? Math.min(...latencies) : 0,
requestLatencyMsMax: requestCount > 0 ? Math.max(...latencies) : 0,
requestLatencyMsAvg: requestCount > 0 ? Math.round(requestLatencyMsTotal / requestCount) : 0,
slowRequestCount: latencies.filter((value) => value >= SLOW_REQUEST_THRESHOLD_MS).length,
slowRequestThresholdMs: SLOW_REQUEST_THRESHOLD_MS,
};
}
function listRunDebugFiles(runId) {
if (!/^\d{8}-\d{6}(?:-\d+)?$/.test(String(runId || ""))) return [];
return fs
.readdirSync(outDir)
.filter((file) => file.startsWith(`${runId}-`) && file.endsWith(".debug.jsonl"))
.sort();
}
function requestLatencySummaryFromRunDebugFiles(runId) {
const totals = {
requestCount: 0,
requestLatencyMsMin: 0,
requestLatencyMsMax: 0,
requestLatencyMsAvg: 0,
slowRequestCount: 0,
slowRequestThresholdMs: SLOW_REQUEST_THRESHOLD_MS,
};
let requestLatencyMsTotal = 0;
for (const file of listRunDebugFiles(runId)) {
const debug = readJsonl(path.join(outDir, file));
const latency = collectRequestLatencySummary(debug.events);
if (latency.requestCount > 0) {
totals.requestLatencyMsMin = totals.requestCount === 0
? latency.requestLatencyMsMin
: Math.min(totals.requestLatencyMsMin, latency.requestLatencyMsMin);
totals.requestLatencyMsMax = Math.max(totals.requestLatencyMsMax, latency.requestLatencyMsMax);
requestLatencyMsTotal += latency.requestLatencyMsAvg * latency.requestCount;
}
totals.requestCount += latency.requestCount;
totals.slowRequestCount += latency.slowRequestCount;
}
totals.requestLatencyMsAvg = totals.requestCount > 0
? Math.round(requestLatencyMsTotal / totals.requestCount)
: 0;
return totals;
}
function hydrateRequestLatencyFromDebugFiles(run) {
if (numberOrZero(run?.requestCount) > 0) return run;
const latency = requestLatencySummaryFromRunDebugFiles(run?.runId);
if (latency.requestCount <= 0) return run;
return {
...run,
...latency,
};
}
function collectRuntimeFingerprint(events) {
const turnEvents = events.filter((event) => event.event === "turn.start");
const sources = turnEvents.length > 0 ? turnEvents : events;
const selectedToolNames = [];
for (const event of sources) {
const names = eventData(event).selectedToolNames;
if (Array.isArray(names)) {
selectedToolNames.push(...names.map(String));
}
}
return {
protocolVersions: uniqueStrings(sources.map((event) => stringMetric(event, "protocolVersion", "protocol_version"))),
selectedToolNameHashes: uniqueStrings(
sources.map((event) => stringMetric(event, "selectedToolNamesHash", "selected_tool_names_hash")),
),
selectedToolNames: uniqueStrings(selectedToolNames),
maxTools: uniqueNumbers(sources.map((event) => numberMetric(event, "maxTools", "max_tools"))),
toolSelectionClipped: uniqueBooleans(
sources.map((event) => booleanMetric(event, "toolSelectionClipped", "tool_selection_clipped")),
),
toolSelectionOmittedCount: uniqueNumbers(
sources.map((event) => numberMetric(event, "toolSelectionOmittedCount", "tool_selection_omitted_count")),
),
toolSelectionValidCount: uniqueNumbers(
sources.map((event) => numberMetric(event, "toolSelectionValidCount", "tool_selection_valid_count")),
),
toolSelectionPromptSources: uniqueStrings(
sources.map((event) => stringMetric(event, "toolSelectionPromptSource", "tool_selection_prompt_source")),
),
toolSelectionPromptChars: uniqueNumbers(
sources.map((event) => numberMetric(event, "toolSelectionPromptChars", "tool_selection_prompt_chars")),
),
toolSelectionUserMessageCount: uniqueNumbers(
sources.map((event) => numberMetric(event, "toolSelectionUserMessageCount", "tool_selection_user_messages")),
),
maxToolResultChars: uniqueNumbers(
sources.map((event) => numberMetric(event, "maxToolResultChars", "max_tool_result_chars")),
),
maxOutputTokens: uniqueNumbers(sources.map((event) => numberMetric(event, "maxOutputTokens", "max_output_tokens"))),
requestTimeoutMs: uniqueNumbers(
sources.map((event) => numberMetric(event, "requestTimeoutMs", "request_timeout_ms")),
),
maxEmptyRetries: uniqueNumbers(
sources.map((event) => numberMetric(event, "maxEmptyRetries", "max_empty_retries")),
),
maxRepairRetries: uniqueNumbers(
sources.map((event) => numberMetric(event, "maxRepairRetries", "max_repair_retries")),
),
maxTotalRecoveries: uniqueNumbers(
sources.map((event) => numberMetric(event, "maxTotalRecoveries", "max_total_recoveries")),
),
};
}
const RUNTIME_FINGERPRINT_FIELD_TYPES = {
protocolVersions: "string",
selectedToolNameHashes: "string",
selectedToolNames: "string",
maxTools: "number",
toolSelectionClipped: "boolean",
toolSelectionOmittedCount: "number",
toolSelectionValidCount: "number",
toolSelectionPromptSources: "string",
toolSelectionPromptChars: "number",
toolSelectionUserMessageCount: "number",
maxToolResultChars: "number",
maxOutputTokens: "number",
requestTimeoutMs: "number",
maxEmptyRetries: "number",
maxRepairRetries: "number",
maxTotalRecoveries: "number",
};
function normalizeFingerprintValues(value, type) {
const values = Array.isArray(value) ? value : [value];
if (type === "number") return uniqueNumbers(values.map((item) => numberOrUndefined(item)));
if (type === "boolean") return uniqueBooleans(values.map((item) => boolOrUndefinedLocal(item)));
return uniqueStrings(values.map((item) => (item === undefined || item === null ? undefined : String(item))));
}
function mergeRuntimeFingerprints(cases) {
const merged = {};
for (const [field, type] of Object.entries(RUNTIME_FINGERPRINT_FIELD_TYPES)) {
const values = [];
for (const item of Array.isArray(cases) ? cases : []) {
const fingerprint = objectOrUndefined(item?.runtimeFingerprint) || {};
const normalized = normalizeFingerprintValues(fingerprint[field], type);
values.push(...normalized);
}
merged[field] = normalizeFingerprintValues(values, type);
}
return merged;
}
function runtimeBoundsFromArtifact(artifact) {
return compactObject({
caseTimeoutSeconds: numberOrUndefined(artifact?.caseTimeoutSeconds),
requestTimeoutMs: numberOrUndefined(artifact?.requestTimeoutMs),
maxOutputTokens: numberOrUndefined(artifact?.maxOutputTokens),
providerHealthPreflightTimeoutMs: numberOrUndefined(artifact?.providerHealthPreflightTimeoutMs),
providerHealthPreflightAttempts: numberOrUndefined(artifact?.providerHealthPreflightAttempts),
providerHealthPreflightRetryDelayMs: numberOrUndefined(artifact?.providerHealthPreflightRetryDelayMs),
stopOnProviderError: boolOrUndefinedLocal(artifact?.stopOnProviderError),
});
}
function providerHealthSummaryFromArtifact(artifact) {
const health = objectOrUndefined(artifact?.providerHealth) || objectOrUndefined(artifact?.providerHealthPreflight);
if (!health) return null;
return compactObject({
schema: typeof health.schema === "string" ? health.schema : undefined,
ok: boolOrUndefinedLocal(health.ok),
provider: typeof health.provider === "string" ? health.provider : undefined,
model: typeof health.model === "string" ? health.model : undefined,
responseModel: typeof health.responseModel === "string" ? health.responseModel : undefined,
httpStatus: numberOrUndefined(health.httpStatus),
timeoutMs: numberOrUndefined(health.timeoutMs),
elapsedMs: numberOrUndefined(health.elapsedMs),
attemptsConfigured: numberOrUndefined(health.attemptsConfigured),
attemptCount: numberOrUndefined(health.attemptCount),
retryCount: numberOrUndefined(health.retryCount),
retryDelayMs: numberOrUndefined(health.retryDelayMs),
errorCode: typeof health.errorCode === "string" ? health.errorCode : undefined,
errorCategory: typeof health.errorCategory === "string" ? health.errorCategory : undefined,
retryable: boolOrUndefinedLocal(health.retryable),
});
}
function providerHealthSignature(summary) {
const health = objectOrUndefined(summary);
if (!health) return null;
const { elapsedMs, ...stableHealth } = health;
return stableHealth;
}
function summarizeCase(debugFileName) {
const debugPath = path.join(outDir, debugFileName);
const mainPath = path.join(outDir, debugFileName.replace(/\.debug\.jsonl$/, ".jsonl"));
const lifecyclePath = path.join(outDir, debugFileName.replace(/\.debug\.jsonl$/, ".lifecycle.json"));
const debug = readJsonl(debugPath);
const main = readJsonl(mainPath);
const lifecycleParsed = fs.existsSync(lifecyclePath) ? readJsonFile(lifecyclePath) : undefined;
const lifecycle = lifecycleParsed?.ok ? objectOrUndefined(lifecycleParsed.value) : undefined;
const recoveryByEvent = {};
const eventByName = {};
const selectedToolCounts = [];
const toolSelectionClippedValues = [];
const toolSelectionOmittedCounts = [];
const toolSelectionValidCounts = [];
let argumentValidationWarningTotal = 0;
const argumentValidationWarningCodeCounts = {};
const toolSelectionReasonCodeCounts = {};
const selectedToolSelectionReasonCodeCounts = {};
const omittedToolSelectionReasonCodeCounts = {};
for (const event of debug.events) {
if (typeof event.event === "string") increment(eventByName, event.event);
if (event.event_category === "recovery" && typeof event.event === "string") {
increment(recoveryByEvent, event.event);
}
if (typeof event.selected_tool_count === "number") {
selectedToolCounts.push(event.selected_tool_count);
}
const selectionClipped = booleanMetric(event, "toolSelectionClipped", "tool_selection_clipped");
if (selectionClipped !== undefined) toolSelectionClippedValues.push(selectionClipped);
const selectionOmittedCount = numberMetric(event, "toolSelectionOmittedCount", "tool_selection_omitted_count");
if (selectionOmittedCount !== undefined) toolSelectionOmittedCounts.push(selectionOmittedCount);
const selectionValidCount = numberMetric(event, "toolSelectionValidCount", "tool_selection_valid_count");
if (selectionValidCount !== undefined) toolSelectionValidCounts.push(selectionValidCount);
const warningCount = argumentValidationWarningCount(event);
if (warningCount > 0) argumentValidationWarningTotal += warningCount;
for (const code of argumentValidationWarningCodes(event)) {
increment(argumentValidationWarningCodeCounts, code);
}
const reasonCodeCounts = collectToolSelectionReasonCodeCounts(event);
for (const [code, count] of Object.entries(reasonCodeCounts.toolSelectionReasonCodes)) {
increment(toolSelectionReasonCodeCounts, code, count);
}
for (const [code, count] of Object.entries(reasonCodeCounts.selectedToolSelectionReasonCodes)) {
increment(selectedToolSelectionReasonCodeCounts, code, count);
}
for (const [code, count] of Object.entries(reasonCodeCounts.omittedToolSelectionReasonCodes)) {
increment(omittedToolSelectionReasonCodeCounts, code, count);
}
}
const toolStartEvents = main.events.filter((event) => event.type === "tool_execution_start");
const errors = main.events.filter(
(event) => event.type === "error" || event.message?.stopReason === "error" || event.message?.errorMessage,
);
const emptyAssistantEnds = main.events.filter(
(event) =>
event.type === "message_end" &&
event.message?.role === "assistant" &&
Array.isArray(event.message.content) &&
event.message.content.length === 0,
).length;
const finalText = finalAssistantText(main.events);
const rawToolMarkupFinalAnswer = isRawToolMarkupFinalAnswer(finalText);
const toolEnvelopeFinalAnswer = isToolEnvelopeOnlyFinalAnswer(finalText);
const finalAnswerQualityOk = finalText.trim().length > 0 && !rawToolMarkupFinalAnswer;
const hasAgentEnd = main.events.some((event) => event.type === "agent_end");
const semanticFlowOk = hasAgentEnd && errors.length === 0 && finalAnswerQualityOk;
const lifecyclePresent = lifecycle !== undefined;
const exitStatus = lifecyclePresent ? numberOrUndefined(lifecycle.exitStatus) : undefined;
const processLifecycleOk = lifecyclePresent ? exitStatus === 0 : undefined;
const elapsedSeconds = lifecyclePresent ? numberOrUndefined(lifecycle.elapsedSeconds) : undefined;
const caseTimeoutSeconds = lifecyclePresent ? numberOrUndefined(lifecycle.caseTimeoutSeconds) : undefined;
const timedOutByWatchdog = lifecycle?.timedOutByWatchdog === true;
const timedOutAfterAgentEnd = timedOutByWatchdog && hasAgentEnd;
const agentEndElapsedSeconds = lifecyclePresent ? numberOrUndefined(lifecycle.agentEndElapsedSeconds) : undefined;
const postAgentEndLingerSeconds =
elapsedSeconds !== undefined && agentEndElapsedSeconds !== undefined
? Math.max(0, elapsedSeconds - agentEndElapsedSeconds)
: undefined;
const runtimeFingerprint = collectRuntimeFingerprint(debug.events);
const providerErrorEvents = debug.events.filter((event) => event.event === "error.provider");
const providerErrorCodes = {};
const providerErrorCategories = {};
const providerHttpStatuses = [];
let retryableProviderErrors = 0;
for (const event of providerErrorEvents) {
increment(providerErrorCodes, stringMetric(event, "errorCode", "error_code") || "unknown_error");
increment(providerErrorCategories, stringMetric(event, "errorCategory", "error_category") || "unknown");
const httpStatus = numberMetric(event, "httpStatus", "http_status");
if (httpStatus !== undefined) providerHttpStatuses.push(httpStatus);
if (booleanMetric(event, "retryable", "retryable") === true) retryableProviderErrors += 1;
}
const requestLatency = collectRequestLatencySummary(debug.events);
return {
...inferCaseParts(debugFileName),
debugFile: debugPath,
mainFile: fs.existsSync(mainPath) ? mainPath : undefined,
lifecycleFile: fs.existsSync(lifecyclePath) ? lifecyclePath : undefined,
debugEvents: debug.events.length,
debugParseErrors: debug.parseErrors,
mainEvents: main.events.length,
mainParseErrors: main.parseErrors,
turns: eventByName["turn.start"] ?? 0,
toolCalls: eventByName.tool_call ?? 0,
recoveries: Object.values(recoveryByEvent).reduce((sum, value) => sum + value, 0),
recoveryByEvent,
piToolStarts: toolStartEvents.map((event) => String(event.toolName || "")),
errors: errors.length,
emptyAssistantEnds,
rawToolMarkupFinalAnswer,
toolEnvelopeFinalAnswer,
finalTextChars: finalText.length,
hasAgentEnd,
finalAnswerQualityOk,
semanticFlowOk,
lifecyclePresent,
exitStatus,
processLifecycleOk,
elapsedSeconds,
caseTimeoutSeconds,
timedOutByWatchdog,
timedOutAfterAgentEnd,
agentEndElapsedSeconds,
postAgentEndLingerSeconds,
runtimeFingerprint,
providerErrors: providerErrorEvents.length,
providerErrorCodes,
providerErrorCategories,
retryableProviderErrors,
providerHttpStatuses: uniqueNumbers(providerHttpStatuses),
argumentValidationWarnings: argumentValidationWarningTotal,
argumentValidationWarningCodes: argumentValidationWarningCodeCounts,
toolSelectionReasonCodes: toolSelectionReasonCodeCounts,
selectedToolSelectionReasonCodes: selectedToolSelectionReasonCodeCounts,
omittedToolSelectionReasonCodes: omittedToolSelectionReasonCodeCounts,
...requestLatency,
selectedToolCountMin: selectedToolCounts.length ? Math.min(...selectedToolCounts) : undefined,
selectedToolCountMax: selectedToolCounts.length ? Math.max(...selectedToolCounts) : undefined,
toolSelectionClipped: toolSelectionClippedValues.length ? toolSelectionClippedValues.includes(true) : undefined,
toolSelectionOmittedCountMin: toolSelectionOmittedCounts.length ? Math.min(...toolSelectionOmittedCounts) : undefined,
toolSelectionOmittedCountMax: toolSelectionOmittedCounts.length ? Math.max(...toolSelectionOmittedCounts) : undefined,
toolSelectionValidCountMin: toolSelectionValidCounts.length ? Math.min(...toolSelectionValidCounts) : undefined,
toolSelectionValidCountMax: toolSelectionValidCounts.length ? Math.max(...toolSelectionValidCounts) : undefined,
};
}
function summarizeSmokeSummaryFile(fileName) {
const file = path.join(outDir, fileName);
const stem = fileName.replace(/-summary\.json$/, "");
const parsed = readJsonFile(file);
if (!parsed.ok) {
return {
file,
runId: stem,
stamp: stem,
ok: false,
parseError: parsed.error,
failures: 0,
debugSummaryStatus: 1,
cases: 0,
recoveries: 0,
recoveryRate: 0,
rawToolMarkupFinalAnswers: 0,
emptyAssistantEnds: 0,
toolEnvelopeFinalAnswers: 0,
errors: 0,
lifecycleArtifacts: 0,
processLifecycleFailures: 0,
watchdogTimeouts: 0,
timedOutAfterAgentEnd: 0,
semanticFlowOkProcessFailures: 0,
postAgentEndLingerMaxSeconds: 0,
providerErrors: 0,
retryableProviderErrors: 0,
requestCount: 0,
requestLatencyMsMin: 0,
requestLatencyMsMax: 0,
requestLatencyMsAvg: 0,
slowRequestCount: 0,
slowRequestThresholdMs: SLOW_REQUEST_THRESHOLD_MS,
argumentValidationWarnings: 0,
toolSelectionClippedCases: 0,
toolSelectionOmittedCountMax: 0,
providerErrorCodes: {},
providerErrorCategories: {},
argumentValidationWarningCodes: {},
toolSelectionReasonCodes: {},
selectedToolSelectionReasonCodes: {},
omittedToolSelectionReasonCodes: {},
selectedCases: [],
caseNames: [],
caseSet: buildCaseSet([]),
runKind: "parse-error",
runtimeBounds: {},
runtimeBoundsSha256: shortSha256({}),
runtimeFingerprint: mergeRuntimeFingerprints([]),
runtimeFingerprintSha256: shortSha256(mergeRuntimeFingerprints([])),
providerHealthSummary: null,
providerHealthSha256: null,
recoveryCaseNames: [],
caseRecoveries: [],
gateFailures: ["summary_parse_error"],
};
}
const artifact = parsed.value;
const totals = artifact?.debugSummary?.totals ?? {};
const cases = Array.isArray(artifact?.debugSummary?.cases) ? artifact.debugSummary.cases : [];
const caseNames = sortedUniqueStrings(cases.map((item) => item?.caseName || "unknown"));
const selectedCases = Array.isArray(artifact?.selectedCases)
? sortedUniqueStrings(artifact.selectedCases)
: caseNames;
const caseSet = normalizeCaseSet(artifact?.caseSet, selectedCases);
const runKind = typeof artifact?.runKind === "string" && artifact.runKind.trim()
? artifact.runKind
: classifyRunKind(caseSet, {
providerHealth: artifact?.providerHealth,
stopReason: artifact?.stopReason,
});
const caseRecoveries = cases
.map((item) => ({
caseName: String(item?.caseName || "unknown"),
recoveries: numberOrZero(item?.recoveries),
}))
.filter((item) => item.recoveries > 0);
const gateFailures = Array.isArray(artifact?.debugSummary?.gateFailures)
? artifact.debugSummary.gateFailures
: [];
const runtimeBounds = runtimeBoundsFromArtifact(artifact);
const runtimeFingerprint = mergeRuntimeFingerprints(cases);
const providerHealthSummary = providerHealthSummaryFromArtifact(artifact);
const providerHealthStableSignature = providerHealthSignature(providerHealthSummary);
return {
file,
schema: artifact?.schema,
ok: artifact?.ok === true,
provider: artifact?.provider,
model: artifact?.model,
stamp: artifact?.stamp || stem,
runId: artifact?.runId || artifact?.stamp || stem,
failures: numberOrZero(artifact?.failures),
debugSummaryStatus: numberOrZero(artifact?.debugSummaryStatus),
cases: numberOrZero(totals.cases),
recoveries: numberOrZero(totals.recoveries),
recoveryRate: numberOrZero(totals.recoveryRate),
rawToolMarkupFinalAnswers: numberOrZero(totals.rawToolMarkupFinalAnswers),
emptyAssistantEnds: numberOrZero(totals.emptyAssistantEnds),
toolEnvelopeFinalAnswers: numberOrZero(totals.toolEnvelopeFinalAnswers),
errors: numberOrZero(totals.errors),
lifecycleArtifacts: numberOrZero(totals.lifecycleArtifacts),
processLifecycleFailures: numberOrZero(totals.processLifecycleFailures),
watchdogTimeouts: numberOrZero(totals.watchdogTimeouts),
timedOutAfterAgentEnd: numberOrZero(totals.timedOutAfterAgentEnd),
semanticFlowOkProcessFailures: numberOrZero(totals.semanticFlowOkProcessFailures),
postAgentEndLingerMaxSeconds: numberOrZero(totals.postAgentEndLingerMaxSeconds),
providerErrors: numberOrZero(totals.providerErrors),
retryableProviderErrors: numberOrZero(totals.retryableProviderErrors),
requestCount: numberOrZero(totals.requestCount),
requestLatencyMsMin: numberOrZero(totals.requestLatencyMsMin),
requestLatencyMsMax: numberOrZero(totals.requestLatencyMsMax),
requestLatencyMsAvg: numberOrZero(totals.requestLatencyMsAvg),
slowRequestCount: numberOrZero(totals.slowRequestCount),
slowRequestThresholdMs: numberOrZero(totals.slowRequestThresholdMs) || SLOW_REQUEST_THRESHOLD_MS,
argumentValidationWarnings: numberOrZero(totals.argumentValidationWarnings),
toolSelectionClippedCases: numberOrZero(totals.toolSelectionClippedCases),
toolSelectionOmittedCountMax: numberOrZero(totals.toolSelectionOmittedCountMax),
providerErrorCodes: objectOrUndefined(totals.providerErrorCodes) || {},
providerErrorCategories: objectOrUndefined(totals.providerErrorCategories) || {},
argumentValidationWarningCodes: objectOrUndefined(totals.argumentValidationWarningCodes) || {},
toolSelectionReasonCodes: objectOrUndefined(totals.toolSelectionReasonCodes) || {},
selectedToolSelectionReasonCodes: objectOrUndefined(totals.selectedToolSelectionReasonCodes) || {},
omittedToolSelectionReasonCodes: objectOrUndefined(totals.omittedToolSelectionReasonCodes) || {},
selectedCases: caseSet.normalizedCases,
caseNames,
caseSet,
runKind,
runtimeBounds,
runtimeBoundsSha256: shortSha256(runtimeBounds),
runtimeFingerprint,
runtimeFingerprintSha256: shortSha256(runtimeFingerprint),
providerHealthSummary,
providerHealthSha256: providerHealthStableSignature ? shortSha256(providerHealthStableSignature) : null,
recoveryCaseNames: caseRecoveries.map((item) => item.caseName),
caseRecoveries,
gateFailures,
};
}
function validateRunId(runId, optionName) {
if (!/^\d{8}-\d{6}(?:-\d+)?$/.test(runId)) {
console.error(`xtalpi-pi-tools debug summary: ${optionName} must look like YYYYMMDD-HHMMSS or YYYYMMDD-HHMMSS-PID`);
process.exit(2);
}
}
function loadSmokeSummaryArtifact(runId, optionName) {
validateRunId(runId, optionName);
const fileName = `${runId}-summary.json`;
const file = path.join(outDir, fileName);
if (!fs.existsSync(file)) {
console.error(`xtalpi-pi-tools debug summary: summary artifact not found for ${optionName}: ${file}`);
process.exit(1);
}
const parsed = readJsonFile(file);
if (!parsed.ok) {
console.error(`xtalpi-pi-tools debug summary: failed to parse ${file}: ${parsed.error}`);
process.exit(1);
}
return {
file,
artifact: parsed.value,
summary: summarizeSmokeSummaryFile(fileName),
};
}
function boolValue(value) {
return value === true;
}
function normalizeCaseForCompare(item) {
return {
caseName: String(item?.caseName || "unknown"),
turns: numberOrZero(item?.turns),
toolCalls: numberOrZero(item?.toolCalls),
recoveries: numberOrZero(item?.recoveries),
emptyAssistantEnds: numberOrZero(item?.emptyAssistantEnds),
rawToolMarkupFinalAnswer: boolValue(item?.rawToolMarkupFinalAnswer),
toolEnvelopeFinalAnswer: boolValue(item?.toolEnvelopeFinalAnswer),
errors: numberOrZero(item?.errors),
finalTextChars: numberOrZero(item?.finalTextChars),
processLifecycleOk: item?.processLifecycleOk === undefined ? undefined : boolValue(item?.processLifecycleOk),
timedOutByWatchdog: boolValue(item?.timedOutByWatchdog),
timedOutAfterAgentEnd: boolValue(item?.timedOutAfterAgentEnd),
semanticFlowOk: boolValue(item?.semanticFlowOk),
elapsedSeconds: item?.elapsedSeconds === undefined ? undefined : numberOrZero(item?.elapsedSeconds),
postAgentEndLingerSeconds: item?.postAgentEndLingerSeconds === undefined
? undefined
: numberOrZero(item?.postAgentEndLingerSeconds),
providerErrors: numberOrZero(item?.providerErrors),
retryableProviderErrors: numberOrZero(item?.retryableProviderErrors),
requestCount: numberOrZero(item?.requestCount),
requestLatencyMsMin: numberOrZero(item?.requestLatencyMsMin),
requestLatencyMsMax: numberOrZero(item?.requestLatencyMsMax),
requestLatencyMsAvg: numberOrZero(item?.requestLatencyMsAvg),
slowRequestCount: numberOrZero(item?.slowRequestCount),
argumentValidationWarnings: numberOrZero(item?.argumentValidationWarnings),
toolSelectionClipped: item?.toolSelectionClipped === undefined ? undefined : boolValue(item?.toolSelectionClipped),
toolSelectionOmittedCountMax: item?.toolSelectionOmittedCountMax === undefined
? undefined
: numberOrZero(item?.toolSelectionOmittedCountMax),
toolSelectionValidCountMax: item?.toolSelectionValidCountMax === undefined
? undefined
: numberOrZero(item?.toolSelectionValidCountMax),
piToolStarts: Array.isArray(item?.piToolStarts) ? item.piToolStarts.map(String) : [],
};
}
function caseMapFromArtifact(artifact) {
const result = new Map();
const cases = Array.isArray(artifact?.debugSummary?.cases) ? artifact.debugSummary.cases : [];
for (const item of cases) {
const normalized = normalizeCaseForCompare(item);
result.set(normalized.caseName, normalized);
}
return result;
}
function deltaNumber(base, head) {
return numberOrZero(head) - numberOrZero(base);
}
function deltaSummary(base, head) {
return {
failures: deltaNumber(base.failures, head.failures),
debugSummaryStatus: deltaNumber(base.debugSummaryStatus, head.debugSummaryStatus),
cases: deltaNumber(base.cases, head.cases),
recoveries: deltaNumber(base.recoveries, head.recoveries),
recoveryRate: numberOrZero(head.recoveryRate) - numberOrZero(base.recoveryRate),
rawToolMarkupFinalAnswers: deltaNumber(base.rawToolMarkupFinalAnswers, head.rawToolMarkupFinalAnswers),
emptyAssistantEnds: deltaNumber(base.emptyAssistantEnds, head.emptyAssistantEnds),
toolEnvelopeFinalAnswers: deltaNumber(base.toolEnvelopeFinalAnswers, head.toolEnvelopeFinalAnswers),
errors: deltaNumber(base.errors, head.errors),
processLifecycleFailures: deltaNumber(base.processLifecycleFailures, head.processLifecycleFailures),
watchdogTimeouts: deltaNumber(base.watchdogTimeouts, head.watchdogTimeouts),
timedOutAfterAgentEnd: deltaNumber(base.timedOutAfterAgentEnd, head.timedOutAfterAgentEnd),
semanticFlowOkProcessFailures: deltaNumber(
base.semanticFlowOkProcessFailures,
head.semanticFlowOkProcessFailures,
),
postAgentEndLingerMaxSeconds: deltaNumber(base.postAgentEndLingerMaxSeconds, head.postAgentEndLingerMaxSeconds),
providerErrors: deltaNumber(base.providerErrors, head.providerErrors),
retryableProviderErrors: deltaNumber(base.retryableProviderErrors, head.retryableProviderErrors),
requestCount: deltaNumber(base.requestCount, head.requestCount),
requestLatencyMsMin: deltaNumber(base.requestLatencyMsMin, head.requestLatencyMsMin),
requestLatencyMsMax: deltaNumber(base.requestLatencyMsMax, head.requestLatencyMsMax),
requestLatencyMsAvg: deltaNumber(base.requestLatencyMsAvg, head.requestLatencyMsAvg),
slowRequestCount: deltaNumber(base.slowRequestCount, head.slowRequestCount),
argumentValidationWarnings: deltaNumber(base.argumentValidationWarnings, head.argumentValidationWarnings),
toolSelectionClippedCases: deltaNumber(base.toolSelectionClippedCases, head.toolSelectionClippedCases),
toolSelectionOmittedCountMax: deltaNumber(base.toolSelectionOmittedCountMax, head.toolSelectionOmittedCountMax),
};
}
function changedNumberMap(base, head, keys) {
const result = {};
for (const key of keys) {
const delta = deltaNumber(base?.[key], head?.[key]);
if (delta !== 0) result[key] = delta;
}
return result;
}
function changedBooleanMap(base, head, keys) {
const result = {};
for (const key of keys) {
const baseValue = boolValue(base?.[key]);
const headValue = boolValue(head?.[key]);
if (baseValue !== headValue) result[key] = headValue;
}
return result;
}
function buildCaseDeltas(baseArtifact, headArtifact) {
const baseCases = caseMapFromArtifact(baseArtifact);
const headCases = caseMapFromArtifact(headArtifact);
const names = [...new Set([...baseCases.keys(), ...headCases.keys()])].sort();
const result = [];
for (const caseName of names) {
const baseCase = baseCases.get(caseName);
const headCase = headCases.get(caseName);
if (!baseCase) {
result.push({ caseName, status: "added", base: null, head: headCase, delta: {} });
continue;
}
if (!headCase) {
result.push({ caseName, status: "removed", base: baseCase, head: null, delta: {} });
continue;
}
const numericDelta = changedNumberMap(baseCase, headCase, [
"turns",
"toolCalls",
"recoveries",
"emptyAssistantEnds",
"errors",
"elapsedSeconds",
"postAgentEndLingerSeconds",
"providerErrors",
"retryableProviderErrors",
"requestCount",
"requestLatencyMsMin",
"requestLatencyMsMax",
"requestLatencyMsAvg",
"slowRequestCount",
"argumentValidationWarnings",
"toolSelectionOmittedCountMax",
"toolSelectionValidCountMax",
]);
const booleanDelta = changedBooleanMap(baseCase, headCase, [
"rawToolMarkupFinalAnswer",
"toolEnvelopeFinalAnswer",
"processLifecycleOk",
"timedOutByWatchdog",
"timedOutAfterAgentEnd",
"semanticFlowOk",
"toolSelectionClipped",
]);
const baseTools = baseCase.piToolStarts.join(",");
const headTools = headCase.piToolStarts.join(",");
const toolDelta = baseTools === headTools ? {} : { piToolStarts: { base: baseCase.piToolStarts, head: headCase.piToolStarts } };
const delta = { ...numericDelta, ...booleanDelta, ...toolDelta };
if (Object.keys(delta).length > 0) {
result.push({ caseName, status: "changed", base: baseCase, head: headCase, delta });
}
}
return result;
}
function printCompare(baseRunId, headRunId) {
const base = loadSmokeSummaryArtifact(baseRunId, "--compare BASE_RUN");
const head = loadSmokeSummaryArtifact(headRunId, "--compare HEAD_RUN");
const delta = deltaSummary(base.summary, head.summary);
const caseDeltas = buildCaseDeltas(base.artifact, head.artifact);
const compare = {
schema: "xtalpi-pi-tools.smoke-compare.v1",
outDir,
baseRunId,
headRunId,
base: base.summary,
head: head.summary,
delta,
okChanged: base.summary.ok !== head.summary.ok,
providerChanged: base.summary.provider !== head.summary.provider,
modelChanged: base.summary.model !== head.summary.model,
caseSetChanged: base.summary.caseSet?.sha256 !== head.summary.caseSet?.sha256,
caseDeltas,
};
if (format === "json") {
console.log(JSON.stringify(compare, null, 2));
} else {
console.log("xtalpi-pi-tools smoke compare");
console.log(`out_dir=${outDir} base=${baseRunId} head=${headRunId}`);
console.log(
`base_ok=${base.summary.ok} head_ok=${head.summary.ok} ok_changed=${compare.okChanged} ` +
`provider_changed=${compare.providerChanged} model_changed=${compare.modelChanged} ` +
`case_set_changed=${compare.caseSetChanged}`,
);
console.log(
`failures_delta=${delta.failures} cases_delta=${delta.cases} recoveries_delta=${delta.recoveries} ` +
`recovery_rate_delta=${delta.recoveryRate.toFixed(4)} ` +
`raw_tool_markup_final_answers_delta=${delta.rawToolMarkupFinalAnswers} ` +
`empty_assistant_ends_delta=${delta.emptyAssistantEnds} ` +
`tool_envelope_final_answers_delta=${delta.toolEnvelopeFinalAnswers} ` +
`errors_delta=${delta.errors} process_lifecycle_failures_delta=${delta.processLifecycleFailures} ` +
`watchdog_timeouts_delta=${delta.watchdogTimeouts} ` +
`timed_out_after_agent_end_delta=${delta.timedOutAfterAgentEnd} ` +
`provider_errors_delta=${delta.providerErrors} retryable_provider_errors_delta=${delta.retryableProviderErrors} ` +
`request_latency_ms_max_delta=${delta.requestLatencyMsMax} slow_requests_delta=${delta.slowRequestCount} ` +
`argument_validation_warnings_delta=${delta.argumentValidationWarnings} ` +
`tool_selection_clipped_cases_delta=${delta.toolSelectionClippedCases} ` +
`tool_selection_omitted_count_max_delta=${delta.toolSelectionOmittedCountMax} ` +
`debug_summary_status_delta=${delta.debugSummaryStatus}`,
);
if (caseDeltas.length === 0) {
console.log("case_deltas=none");
} else {
console.log("case_deltas:");
for (const item of caseDeltas) {
console.log(`- ${item.caseName}: status=${item.status} delta=${JSON.stringify(item.delta)}`);
}
}
}
process.exit(0);
}
function listSmokeSummaryFiles() {
return fs.readdirSync(outDir)
.filter((file) => /^\d{8}-\d{6}(?:-\d+)?-summary\.json$/.test(file))
.sort();
}
function buildHistory(limit) {
const summaryFiles = listSmokeSummaryFiles();
if (summaryFiles.length === 0) {
console.error(`xtalpi-pi-tools debug summary: no *-summary.json files found in ${outDir}`);
process.exit(1);
}
let candidateArtifacts = summaryFiles.length;
let filteredOutArtifacts = 0;
let runs;
if (runKindFilter !== undefined) {
const newestRuns = summaryFiles.slice().reverse().map(summarizeSmokeSummaryFile);
const candidateRuns = newestRuns.filter((run) => runKindMatches(run.runKind, runKindFilter));
candidateArtifacts = candidateRuns.length;
filteredOutArtifacts = summaryFiles.length - candidateRuns.length;
runs = candidateRuns.slice(0, limit).map(hydrateRequestLatencyFromDebugFiles);
} else {
const selectedFiles = summaryFiles.slice(-limit).reverse();
runs = selectedFiles.map(summarizeSmokeSummaryFile).map(hydrateRequestLatencyFromDebugFiles);
}
const parseErrorCount = runs.filter((run) => run.parseError).length;
return {
schema: "xtalpi-pi-tools.smoke-history.v1",
outDir,
requested: limit,
totalArtifacts: summaryFiles.length,
candidateArtifacts,
filteredOutArtifacts,
filter: {
runKinds: runKindFilter || null,
},
found: runs.length,
order: "newest_first",
parseErrorCount,
runs,
};
}
function countBy(runs, valueFn) {
const counts = {};
for (const run of runs) {
const value = valueFn(run) || "unknown";
counts[value] = (counts[value] ?? 0) + 1;
}
return counts;
}
function buildSignatureGroups(runs, signatureFn, detailsFn) {
const groups = new Map();
for (const run of runs) {
const signature = signatureFn(run);
const key = signature === undefined || signature === null || signature === "" ? "unknown" : String(signature);
if (!groups.has(key)) {
groups.set(key, {
key,
count: 0,
runIds: [],
...detailsFn(run),
});
}
const group = groups.get(key);
group.count += 1;
group.runIds.push(run.runId);
}
return [...groups.values()].sort((left, right) => {
if (right.count !== left.count) return right.count - left.count;
return left.key.localeCompare(right.key);
});
}
function aggregateDriftSignals(runs) {
const totals = {
failures: 0,
recoveries: 0,
maxRecoveryRate: 0,
rawToolMarkupFinalAnswers: 0,
emptyAssistantEnds: 0,
toolEnvelopeFinalAnswers: 0,
errors: 0,
processLifecycleFailures: 0,
watchdogTimeouts: 0,
timedOutAfterAgentEnd: 0,
providerErrors: 0,
retryableProviderErrors: 0,
requestCount: 0,
requestLatencyMsMax: 0,
slowRequestCount: 0,
argumentValidationWarnings: 0,
};
const providerErrorCodes = {};
const providerErrorCategories = {};
const argumentValidationWarningCodes = {};
const toolSelectionReasonCodes = {};
const selectedToolSelectionReasonCodes = {};
const omittedToolSelectionReasonCodes = {};
for (const run of runs) {
totals.failures += numberOrZero(run.failures);
totals.recoveries += numberOrZero(run.recoveries);
totals.maxRecoveryRate = Math.max(totals.maxRecoveryRate, numberOrZero(run.recoveryRate));
totals.rawToolMarkupFinalAnswers += numberOrZero(run.rawToolMarkupFinalAnswers);
totals.emptyAssistantEnds += numberOrZero(run.emptyAssistantEnds);
totals.toolEnvelopeFinalAnswers += numberOrZero(run.toolEnvelopeFinalAnswers);
totals.errors += numberOrZero(run.errors);
totals.processLifecycleFailures += numberOrZero(run.processLifecycleFailures);
totals.watchdogTimeouts += numberOrZero(run.watchdogTimeouts);
totals.timedOutAfterAgentEnd += numberOrZero(run.timedOutAfterAgentEnd);
totals.providerErrors += numberOrZero(run.providerErrors);
totals.retryableProviderErrors += numberOrZero(run.retryableProviderErrors);
totals.requestCount += numberOrZero(run.requestCount);
totals.requestLatencyMsMax = Math.max(totals.requestLatencyMsMax, numberOrZero(run.requestLatencyMsMax));
totals.slowRequestCount += numberOrZero(run.slowRequestCount);
totals.argumentValidationWarnings += numberOrZero(run.argumentValidationWarnings);
for (const [code, count] of Object.entries(objectOrUndefined(run.providerErrorCodes) || {})) {
increment(providerErrorCodes, code, count);
}
for (const [category, count] of Object.entries(objectOrUndefined(run.providerErrorCategories) || {})) {
increment(providerErrorCategories, category, count);
}
for (const [code, count] of Object.entries(objectOrUndefined(run.argumentValidationWarningCodes) || {})) {
increment(argumentValidationWarningCodes, code, count);
}
for (const [code, count] of Object.entries(objectOrUndefined(run.toolSelectionReasonCodes) || {})) {
increment(toolSelectionReasonCodes, code, count);
}
for (const [code, count] of Object.entries(objectOrUndefined(run.selectedToolSelectionReasonCodes) || {})) {
increment(selectedToolSelectionReasonCodes, code, count);
}
for (const [code, count] of Object.entries(objectOrUndefined(run.omittedToolSelectionReasonCodes) || {})) {
increment(omittedToolSelectionReasonCodes, code, count);
}
}
return {
...totals,
providerErrorCodes,
providerErrorCategories,
argumentValidationWarningCodes,
toolSelectionReasonCodes,
selectedToolSelectionReasonCodes,
omittedToolSelectionReasonCodes,
};
}
function compactDriftRun(run) {
return {
runId: run.runId,
ok: run.ok,
provider: run.provider || null,
model: run.model || null,
runKind: run.runKind || null,
cases: run.cases,
selectedCases: run.selectedCases,
caseSetSha256: run.caseSet?.sha256 || null,
failures: run.failures,
recoveries: run.recoveries,
recoveryRate: run.recoveryRate,
rawToolMarkupFinalAnswers: run.rawToolMarkupFinalAnswers,
emptyAssistantEnds: run.emptyAssistantEnds,
toolEnvelopeFinalAnswers: run.toolEnvelopeFinalAnswers,
errors: run.errors,
processLifecycleFailures: run.processLifecycleFailures,
watchdogTimeouts: run.watchdogTimeouts,
timedOutAfterAgentEnd: run.timedOutAfterAgentEnd,
providerErrors: run.providerErrors,
retryableProviderErrors: run.retryableProviderErrors,
requestCount: run.requestCount,
requestLatencyMsMax: run.requestLatencyMsMax,
requestLatencyMsAvg: run.requestLatencyMsAvg,
slowRequestCount: run.slowRequestCount,
slowRequestThresholdMs: run.slowRequestThresholdMs,
providerErrorCodes: run.providerErrorCodes,
providerErrorCategories: run.providerErrorCategories,
argumentValidationWarnings: run.argumentValidationWarnings,
argumentValidationWarningCodes: run.argumentValidationWarningCodes,
toolSelectionReasonCodes: run.toolSelectionReasonCodes,
selectedToolSelectionReasonCodes: run.selectedToolSelectionReasonCodes,
omittedToolSelectionReasonCodes: run.omittedToolSelectionReasonCodes,
runtimeBounds: run.runtimeBounds,
runtimeBoundsSha256: run.runtimeBoundsSha256,
runtimeFingerprint: run.runtimeFingerprint,
runtimeFingerprintSha256: run.runtimeFingerprintSha256,
providerHealth: run.providerHealthSummary,
providerHealthSha256: run.providerHealthSha256,
parseError: run.parseError || null,
};
}
function classifyArtifactFile(file) {
let match = file.match(/^(\d{8}-\d{6}(?:-\d+)?)-summary\.json$/);
if (match) return { runId: match[1], kind: "summary", caseName: null };
match = file.match(/^(\d{8}-\d{6}(?:-\d+)?)-debug-summary\.json$/);
if (match) return { runId: match[1], kind: "debug-summary", caseName: null };
match = file.match(/^(\d{8}-\d{6}(?:-\d+)?)-provider-health\.json$/);
if (match) return { runId: match[1], kind: "provider-health", caseName: null };
match = file.match(/^(\d{8}-\d{6}(?:-\d+)?)-(.+)\.debug\.jsonl$/);
if (match) return { runId: match[1], kind: "case-debug", caseName: match[2] };
match = file.match(/^(\d{8}-\d{6}(?:-\d+)?)-(.+)\.lifecycle\.json$/);
if (match) return { runId: match[1], kind: "case-lifecycle", caseName: match[2] };
match = file.match(/^(\d{8}-\d{6}(?:-\d+)?)-(.+)\.jsonl$/);
if (match) return { runId: match[1], kind: "case-events", caseName: match[2] };
match = file.match(/^(\d{8}-\d{6}(?:-\d+)?)-(.+)\.stderr$/);
if (match) return { runId: match[1], kind: "case-stderr", caseName: match[2] };
match = file.match(/^(\d{8}-\d{6}(?:-\d+)?)-(.+)\.txt$/);
if (match) return { runId: match[1], kind: "case-text", caseName: match[2] };
return { runId: null, kind: "unknown", caseName: null };
}
function listArtifactFiles() {
return fs.readdirSync(outDir, { withFileTypes: true })
.filter((entry) => entry.isFile())
.map((entry) => {
const file = entry.name;
const stat = fs.statSync(path.join(outDir, file));
return {
file,
bytes: stat.size,
...classifyArtifactFile(file),
};
})
.sort((left, right) => left.file.localeCompare(right.file));
}
function summarizeFileGroup(files) {
return {
fileCount: files.length,
bytes: files.reduce((sum, item) => sum + numberOrZero(item.bytes), 0),
kinds: countBy(files, (item) => item.kind),
sampleFiles: files.slice(0, retentionPolicy.sampleLimit).map((item) => item.file),
};
}
function hasQualitySignal(run) {
if (run.parseError) return true;
if (run.ok !== true) return true;
return [
"failures",
"debugSummaryStatus",
"recoveries",
"rawToolMarkupFinalAnswers",
"emptyAssistantEnds",
"toolEnvelopeFinalAnswers",
"errors",
"processLifecycleFailures",
"watchdogTimeouts",
"timedOutAfterAgentEnd",
"providerErrors",
"retryableProviderErrors",
"slowRequestCount",
"argumentValidationWarnings",
].some((field) => numberOrZero(run[field]) > 0);
}
function retentionLimitForRunKind(runKind) {
if (runKind === "full-suite") return retentionPolicy.keepFullSuite;
if (runKind === "targeted") return retentionPolicy.keepTargeted;
if (runKind === "preflight-failed") return retentionPolicy.keepPreflightFailed;
if (runKind === "empty") return retentionPolicy.keepEmpty;
return 0;
}
function addRetainReason(retainReasons, runId, reason) {
if (!retainReasons.has(runId)) retainReasons.set(runId, []);
retainReasons.get(runId).push(reason);
}
function compactRetentionRun(run, groupedFiles, retainReasons) {
const files = groupedFiles.get(run.runId) || [];
return {
runId: run.runId,
ok: run.ok,
runKind: run.runKind || null,
cases: run.cases,
failures: run.failures,
recoveries: run.recoveries,
debugSummaryStatus: run.debugSummaryStatus,
rawToolMarkupFinalAnswers: run.rawToolMarkupFinalAnswers,
emptyAssistantEnds: run.emptyAssistantEnds,
toolEnvelopeFinalAnswers: run.toolEnvelopeFinalAnswers,
errors: run.errors,
processLifecycleFailures: run.processLifecycleFailures,
watchdogTimeouts: run.watchdogTimeouts,
timedOutAfterAgentEnd: run.timedOutAfterAgentEnd,
providerErrors: run.providerErrors,
retryableProviderErrors: run.retryableProviderErrors,
requestCount: run.requestCount,
requestLatencyMsMax: run.requestLatencyMsMax,
requestLatencyMsAvg: run.requestLatencyMsAvg,
slowRequestCount: run.slowRequestCount,
slowRequestThresholdMs: run.slowRequestThresholdMs,
argumentValidationWarnings: run.argumentValidationWarnings,
qualitySignalPresent: hasQualitySignal(run),
parseError: run.parseError || null,
retainReasons: retainReasons.get(run.runId) || [],
fileCount: files.length,
bytes: files.reduce((sum, item) => sum + numberOrZero(item.bytes), 0),
artifactKinds: countBy(files, (item) => item.kind),
};
}
function buildRetentionReport() {
const files = listArtifactFiles();
const groupedFiles = new Map();
for (const file of files) {
if (!file.runId) continue;
if (!groupedFiles.has(file.runId)) groupedFiles.set(file.runId, []);
groupedFiles.get(file.runId).push(file);
}
const summaryFiles = listSmokeSummaryFiles();
if (summaryFiles.length === 0) {
console.error(`xtalpi-pi-tools debug summary: no *-summary.json files found in ${outDir}`);
process.exit(1);
}
const runs = summaryFiles.slice().reverse().map(summarizeSmokeSummaryFile);
const summaryRunIds = new Set(runs.map((run) => run.runId));
const retainReasons = new Map();
for (const run of runs) {
if (hasQualitySignal(run)) {
addRetainReason(retainReasons, run.runId, "quality_signal");
}
}
const runKindCounts = countBy(runs, (run) => run.runKind);
const runsByKind = new Map();
for (const run of runs) {
const kind = run.runKind || "unknown";
if (!runsByKind.has(kind)) runsByKind.set(kind, []);
runsByKind.get(kind).push(run);
}
for (const [kind, kindRuns] of runsByKind.entries()) {
const limit = retentionLimitForRunKind(kind);
for (const run of kindRuns.slice(0, limit)) {
addRetainReason(retainReasons, run.runId, `latest_${kind}_within_limit`);
}
}
const retainedRuns = runs.filter((run) => retainReasons.has(run.runId));
const archiveCandidates = runs.filter((run) => !retainReasons.has(run.runId));
const archiveCandidateRunIds = archiveCandidates.map((run) => run.runId);
const archiveCandidateRunIdSet = new Set(archiveCandidateRunIds);
const archiveCandidateFiles = files.filter((file) => file.runId && archiveCandidateRunIdSet.has(file.runId));
const runsWithoutSummary = [...groupedFiles.entries()]
.filter(([runId]) => !summaryRunIds.has(runId))
.map(([runId, groupFiles]) => ({
runId,
...summarizeFileGroup(groupFiles),
}))
.sort((left, right) => right.runId.localeCompare(left.runId));
const unknownFiles = files
.filter((file) => !file.runId)
.map((file) => ({ file: file.file, bytes: file.bytes, kind: file.kind }));
return {
schema: "xtalpi-pi-tools.smoke-retention-report.v1",
outDir,
policy: retentionPolicy,
totals: {
totalFiles: files.length,
totalBytes: files.reduce((sum, item) => sum + numberOrZero(item.bytes), 0),
summaryArtifacts: summaryFiles.length,
runsWithSummary: runs.length,
retainedRuns: retainedRuns.length,
archiveCandidateRuns: archiveCandidates.length,
archiveCandidateFiles: archiveCandidateFiles.length,
archiveCandidateBytes: archiveCandidateFiles.reduce((sum, item) => sum + numberOrZero(item.bytes), 0),
runsWithoutSummary: runsWithoutSummary.length,
unknownFiles: unknownFiles.length,
},
fileKindCounts: countBy(files, (item) => item.kind),
runKindCounts,
retainedRunIds: retainedRuns.map((run) => run.runId),
archiveCandidateRunIds,
archiveCandidateSample: archiveCandidates
.slice(0, retentionPolicy.sampleLimit)
.map((run) => compactRetentionRun(run, groupedFiles, retainReasons)),
retainedRunSample: retainedRuns
.slice(0, retentionPolicy.sampleLimit)
.map((run) => compactRetentionRun(run, groupedFiles, retainReasons)),
qualitySignalRunIds: runs.filter(hasQualitySignal).map((run) => run.runId),
runsWithoutSummary: runsWithoutSummary.slice(0, retentionPolicy.sampleLimit),
unknownFiles: unknownFiles.slice(0, retentionPolicy.sampleLimit),
};
}
function printRetentionReport() {
const report = buildRetentionReport();
if (format === "json") {
console.log(JSON.stringify(report, null, 2));
} else {
console.log("xtalpi-pi-tools smoke retention report");
console.log(
`out_dir=${outDir} report_only=true summary_artifacts=${report.totals.summaryArtifacts} ` +
`total_files=${report.totals.totalFiles} total_bytes=${report.totals.totalBytes}`,
);
console.log(
`policy=${JSON.stringify(report.policy)} run_kinds=${JSON.stringify(report.runKindCounts)} ` +
`file_kinds=${JSON.stringify(report.fileKindCounts)}`,
);
console.log(
`retained_runs=${report.totals.retainedRuns} archive_candidate_runs=${report.totals.archiveCandidateRuns} ` +
`archive_candidate_files=${report.totals.archiveCandidateFiles} ` +
`archive_candidate_bytes=${report.totals.archiveCandidateBytes}`,
);
console.log(
`runs_without_summary=${report.totals.runsWithoutSummary} unknown_files=${report.totals.unknownFiles} ` +
`quality_signal_runs=${report.qualitySignalRunIds.length}`,
);
if (report.archiveCandidateSample.length > 0) {
console.log(
`archive_candidates_sample showing=${report.archiveCandidateSample.length}/${report.totals.archiveCandidateRuns}:`,
);
for (const run of report.archiveCandidateSample) {
console.log(
`- ${run.runId}: run_kind=${run.runKind || "(missing)"} ok=${run.ok} cases=${run.cases} ` +
`failures=${run.failures} recoveries=${run.recoveries} provider_errors=${run.providerErrors} ` +
`files=${run.fileCount} bytes=${run.bytes}`,
);
}
}
}
}
function buildDrift(limit) {
const history = buildHistory(limit);
const runs = history.runs;
const runKindCounts = countBy(runs, (run) => run.runKind);
const providerModels = buildSignatureGroups(
runs,
(run) => `${run.provider || "(missing)"}\u0000${run.model || "(missing)"}`,
(run) => ({
provider: run.provider || null,
model: run.model || null,
}),
);
const caseSets = buildSignatureGroups(
runs,
(run) => run.caseSet?.sha256,
(run) => ({
sha256: run.caseSet?.sha256 || null,
countCases: run.caseSet?.count ?? run.cases,
canonical: run.caseSet?.canonical || null,
}),
);
const runtimeFingerprints = buildSignatureGroups(
runs,
(run) => run.runtimeFingerprintSha256,
(run) => ({
sha256: run.runtimeFingerprintSha256,
fingerprint: run.runtimeFingerprint,
}),
);
const runtimeBounds = buildSignatureGroups(
runs,
(run) => run.runtimeBoundsSha256,
(run) => ({
sha256: run.runtimeBoundsSha256,
bounds: run.runtimeBounds,
}),
);
const providerHealth = buildSignatureGroups(
runs,
(run) => run.providerHealthSha256,
(run) => ({
sha256: run.providerHealthSha256,
summary: run.providerHealthSummary,
}),
);
const qualityTotals = aggregateDriftSignals(runs);
const warnings = [];
if (history.found < history.requested) {
warnings.push(`requested ${history.requested} runs but found ${history.found}`);
}
if (history.parseErrorCount > 0) {
warnings.push(`selected summaries include ${history.parseErrorCount} parse error(s)`);
}
return {
schema: "xtalpi-pi-tools.smoke-drift.v1",
outDir,
requested: history.requested,
totalArtifacts: history.totalArtifacts,
candidateArtifacts: history.candidateArtifacts,
filteredOutArtifacts: history.filteredOutArtifacts,
filter: history.filter,
found: history.found,
order: history.order,
parseErrorCount: history.parseErrorCount,
latestRunId: runs[0]?.runId || null,
baselineRunId: runs.at(-1)?.runId || null,
runKindCounts,
dimensions: {
providerModels,
caseSets,
runtimeFingerprints,
runtimeBounds,
providerHealth,
},
drift: {
providerModelChanged: providerModels.length > 1,
runKindChanged: Object.keys(runKindCounts).length > 1,
caseSetChanged: caseSets.length > 1,
runtimeFingerprintChanged: runtimeFingerprints.length > 1,
runtimeBoundsChanged: runtimeBounds.length > 1,
providerHealthChanged: providerHealth.length > 1,
qualitySignalsPresent: [
"failures",
"recoveries",
"rawToolMarkupFinalAnswers",
"emptyAssistantEnds",
"toolEnvelopeFinalAnswers",
"errors",
"processLifecycleFailures",
"watchdogTimeouts",
"timedOutAfterAgentEnd",
"providerErrors",
"retryableProviderErrors",
"slowRequestCount",
"argumentValidationWarnings",
].some((field) => numberOrZero(qualityTotals[field]) > 0),
},
qualityTotals,
warnings,
runs: runs.map(compactDriftRun),
};
}
function printDrift(limit) {
const drift = buildDrift(limit);
if (format === "json") {
console.log(JSON.stringify(drift, null, 2));
} else {
console.log("xtalpi-pi-tools smoke drift");
console.log(
`out_dir=${outDir} requested=${drift.requested} found=${drift.found} ` +
`total_artifacts=${drift.totalArtifacts} candidate_artifacts=${drift.candidateArtifacts} ` +
`filtered_out_artifacts=${drift.filteredOutArtifacts} ` +
`run_kind_filter=${drift.filter.runKinds ? drift.filter.runKinds.join(",") : "(none)"} order=${drift.order}`,
);
console.log(
`latest=${drift.latestRunId || "(none)"} baseline=${drift.baselineRunId || "(none)"} ` +
`provider_model_changed=${drift.drift.providerModelChanged} run_kind_changed=${drift.drift.runKindChanged} ` +
`case_set_changed=${drift.drift.caseSetChanged} runtime_fingerprint_changed=${drift.drift.runtimeFingerprintChanged} ` +
`runtime_bounds_changed=${drift.drift.runtimeBoundsChanged} provider_health_changed=${drift.drift.providerHealthChanged} ` +
`quality_signals_present=${drift.drift.qualitySignalsPresent}`,
);
console.log(
`run_kinds=${JSON.stringify(drift.runKindCounts)} ` +
`provider_models=${JSON.stringify(drift.dimensions.providerModels.map((item) => ({
provider: item.provider,
model: item.model,
count: item.count,
})))} ` +
`case_sets=${JSON.stringify(drift.dimensions.caseSets.map((item) => ({
sha256: item.sha256,
count: item.count,
countCases: item.countCases,
})))} ` +
`runtime_fingerprints=${JSON.stringify(drift.dimensions.runtimeFingerprints.map((item) => ({
sha256: item.sha256,
count: item.count,
})))} ` +
`runtime_bounds=${JSON.stringify(drift.dimensions.runtimeBounds.map((item) => ({
sha256: item.sha256,
count: item.count,
bounds: item.bounds,
})))}`,
);
console.log(
`quality_totals=${JSON.stringify({
failures: drift.qualityTotals.failures,
recoveries: drift.qualityTotals.recoveries,
rawToolMarkupFinalAnswers: drift.qualityTotals.rawToolMarkupFinalAnswers,
emptyAssistantEnds: drift.qualityTotals.emptyAssistantEnds,
toolEnvelopeFinalAnswers: drift.qualityTotals.toolEnvelopeFinalAnswers,
errors: drift.qualityTotals.errors,
processLifecycleFailures: drift.qualityTotals.processLifecycleFailures,
providerErrors: drift.qualityTotals.providerErrors,
retryableProviderErrors: drift.qualityTotals.retryableProviderErrors,
requestCount: drift.qualityTotals.requestCount,
requestLatencyMsMax: drift.qualityTotals.requestLatencyMsMax,
slowRequestCount: drift.qualityTotals.slowRequestCount,
argumentValidationWarnings: drift.qualityTotals.argumentValidationWarnings,
toolSelectionReasonCodes: drift.qualityTotals.toolSelectionReasonCodes,
selectedToolSelectionReasonCodes: drift.qualityTotals.selectedToolSelectionReasonCodes,
omittedToolSelectionReasonCodes: drift.qualityTotals.omittedToolSelectionReasonCodes,
})}`,
);
for (const run of drift.runs) {
const providerHealthText = run.providerHealth
? ` provider_health_ok=${run.providerHealth.ok ?? "(unknown)"} provider_health_status=${run.providerHealth.httpStatus ?? "(none)"}`
: " provider_health=(missing)";
console.log(
`- ${run.runId}: provider=${run.provider || "(missing)"} model=${run.model || "(missing)"} ` +
`run_kind=${run.runKind || "(missing)"} ok=${run.ok} cases=${run.cases} case_set_sha256=${run.caseSetSha256 || "(missing)"} ` +
`runtime_fingerprint_sha256=${run.runtimeFingerprintSha256 || "(missing)"} ` +
`runtime_bounds_sha256=${run.runtimeBoundsSha256 || "(missing)"} ` +
`recoveries=${run.recoveries} recovery_rate=${numberOrZero(run.recoveryRate).toFixed(4)} ` +
`provider_errors=${run.providerErrors} retryable_provider_errors=${run.retryableProviderErrors} ` +
`request_latency_ms_max=${run.requestLatencyMsMax} slow_requests=${run.slowRequestCount} ` +
`argument_validation_warnings=${run.argumentValidationWarnings} raw_tool_markup_final_answers=${run.rawToolMarkupFinalAnswers} ` +
`empty_assistant_ends=${run.emptyAssistantEnds} process_lifecycle_failures=${run.processLifecycleFailures}` +
providerHealthText,
);
}
if (drift.warnings.length > 0) {
console.error(`warnings=${JSON.stringify(drift.warnings)}`);
}
}
process.exit(drift.parseErrorCount > 0 ? 1 : 0);
}
function buildRunKindRequirementFailures(runs) {
if (requireRunKinds === undefined) return [];
return runs
.filter((run) => !runKindMatches(run.runKind, requireRunKinds))
.map((run) => `${run.runId}: expected run_kind=${formatRunKinds(requireRunKinds)}, got ${run.runKind || "(missing)"}`);
}
function printHistory(limit) {
const history = buildHistory(limit);
const { runs } = history;
const runKindFailures = buildRunKindRequirementFailures(runs);
if (runKindFailures.length > 0) {
history.gateFailures = runKindFailures;
}
if (format === "json") {
console.log(JSON.stringify(history, null, 2));
} else {
console.log("xtalpi-pi-tools smoke history");
console.log(
`out_dir=${outDir} requested=${history.requested} found=${history.found} total_artifacts=${history.totalArtifacts} ` +
`candidate_artifacts=${history.candidateArtifacts} filtered_out_artifacts=${history.filteredOutArtifacts} ` +
`run_kind_filter=${history.filter.runKinds ? history.filter.runKinds.join(",") : "(none)"} order=newest_first`,
);
for (const run of runs) {
const parseText = run.parseError ? ` parse_error=${JSON.stringify(run.parseError)}` : "";
const gateText = run.gateFailures?.length ? ` gate_failures=${JSON.stringify(run.gateFailures)}` : "";
const providerText = run.provider ? ` provider=${run.provider}` : "";
const modelText = run.model ? ` model=${run.model}` : "";
const selectedText = run.selectedCases?.length ? ` selected_cases=${run.selectedCases.join(",")}` : "";
const caseSetText = run.caseSet?.sha256 ? ` case_set_sha256=${run.caseSet.sha256}` : "";
const runKindText = run.runKind ? ` run_kind=${run.runKind}` : "";
const reasonCodesText = Object.keys(run.toolSelectionReasonCodes || {}).length
? ` tool_selection_reason_codes=${JSON.stringify(run.toolSelectionReasonCodes)}` +
` selected_tool_selection_reason_codes=${JSON.stringify(run.selectedToolSelectionReasonCodes || {})}` +
` omitted_tool_selection_reason_codes=${JSON.stringify(run.omittedToolSelectionReasonCodes || {})}`
: "";
console.log(
`- ${run.runId}: ok=${run.ok} failures=${run.failures} cases=${run.cases} ` +
`recoveries=${run.recoveries} recovery_rate=${run.recoveryRate.toFixed(4)} ` +
`raw_tool_markup_final_answers=${run.rawToolMarkupFinalAnswers} ` +
`empty_assistant_ends=${run.emptyAssistantEnds} ` +
`tool_envelope_final_answers=${run.toolEnvelopeFinalAnswers} ` +
`errors=${run.errors} process_lifecycle_failures=${run.processLifecycleFailures} ` +
`watchdog_timeouts=${run.watchdogTimeouts} timed_out_after_agent_end=${run.timedOutAfterAgentEnd} ` +
`provider_errors=${run.providerErrors} retryable_provider_errors=${run.retryableProviderErrors} ` +
`request_latency_ms=${run.requestLatencyMsMax}/${run.requestLatencyMsAvg}/${run.requestCount} ` +
`slow_requests=${run.slowRequestCount} ` +
`argument_validation_warnings=${run.argumentValidationWarnings} ` +
`tool_selection_clipped_cases=${run.toolSelectionClippedCases} ` +
`tool_selection_omitted_count_max=${run.toolSelectionOmittedCountMax} ` +
`debug_summary_status=${run.debugSummaryStatus}` +
`${providerText}${modelText}${runKindText}${selectedText}${caseSetText}${reasonCodesText}${gateText}${parseText}`,
);
}
if (runKindFailures.length > 0) {
console.error(`gate_failures=${JSON.stringify(runKindFailures)}`);
}
}
process.exit(history.parseErrorCount > 0 || runKindFailures.length > 0 ? 1 : 0);
}
function buildRecoveryTrend(runs) {
const latest = runs[0];
const previous = runs[1];
const latestRecoveries = numberOrZero(latest?.recoveries);
const previousRecoveries = numberOrZero(previous?.recoveries);
const latestRecoveryRate = numberOrZero(latest?.recoveryRate);
const previousRecoveryRate = numberOrZero(previous?.recoveryRate);
return {
latestRunId: latest?.runId,
previousRunId: previous?.runId,
latestRecoveries,
previousRecoveries,
recoveryDelta: previous ? latestRecoveries - previousRecoveries : 0,
latestRecoveryRate,
previousRecoveryRate,
recoveryRateDelta: previous ? latestRecoveryRate - previousRecoveryRate : 0,
};
}
function buildRecoveryCaseRunCounts(runs) {
const result = {};
for (const run of runs) {
const names = new Set(Array.isArray(run.recoveryCaseNames) ? run.recoveryCaseNames : []);
for (const name of names) {
result[name] = (result[name] ?? 0) + 1;
}
}
return result;
}
function presentReasonCodes(counts) {
return Object.entries(objectOrUndefined(counts) || {})
.filter(([, count]) => numberOrZero(count) > 0)
.map(([code]) => code)
.sort();
}
function missingReasonCodes(counts, required) {
if (!Array.isArray(required)) return [];
const present = new Set(presentReasonCodes(counts));
return required.filter((code) => !present.has(code));
}
function forbiddenReasonCodes(counts, forbidden) {
if (!Array.isArray(forbidden)) return [];
const present = new Set(presentReasonCodes(counts));
return forbidden.filter((code) => present.has(code));
}
function buildReasonCodeGateFailures(subject, label, counts, required, forbidden) {
const failures = [];
const missing = missingReasonCodes(counts, required);
if (missing.length > 0) {
failures.push(
`${subject}: expected ${label} to include ${required.join(",")}, missing ${missing.join(",")}`,
);
}
const presentForbidden = forbiddenReasonCodes(counts, forbidden);
if (presentForbidden.length > 0) {
failures.push(
`${subject}: expected ${label} not to include ${forbidden.join(",")}, got ${presentForbidden.join(",")}`,
);
}
return failures;
}
function buildRuntimeStability(runs) {
return {
runtimeFingerprints: buildSignatureGroups(
runs,
(run) => run.runtimeFingerprintSha256,
(run) => ({
sha256: run.runtimeFingerprintSha256,
fingerprint: run.runtimeFingerprint,
}),
),
runtimeBounds: buildSignatureGroups(
runs,
(run) => run.runtimeBoundsSha256,
(run) => ({
sha256: run.runtimeBoundsSha256,
bounds: run.runtimeBounds,
}),
),
};
}
function formatSignatureGroups(groups) {
return groups
.map((group) => `${group.sha256 || group.key || "unknown"}:${group.runIds.join(",")}`)
.join("; ");
}
function evaluateTrendGate(history) {
const hardLimits = {
profile: gates.profile,
expectCases: gates.expectCases,
expectCaseNames: gates.expectCaseNames,
runKindFilter: gates.runKindFilter,
requireRunKinds: gates.requireRunKinds,
maxErrors: gates.maxErrors ?? 0,
maxEmptyAssistantEnds: gates.maxEmptyAssistantEnds ?? 0,
maxRawToolMarkupFinalAnswers: gates.maxRawToolMarkupFinalAnswers ?? 0,
maxToolEnvelopeFinalAnswers: gates.maxRawToolMarkupFinalAnswers ?? 0,
maxRecoveries: gates.maxRecoveries,
maxRecoveryRate: gates.maxRecoveryRate,
maxRequestLatencyMs: gates.maxRequestLatencyMs,
maxSlowRequests: gates.maxSlowRequests,
requireToolSelectionReasonCodes: gates.requireToolSelectionReasonCodes,
requireSelectedToolSelectionReasonCodes: gates.requireSelectedToolSelectionReasonCodes,
requireOmittedToolSelectionReasonCodes: gates.requireOmittedToolSelectionReasonCodes,
forbidToolSelectionReasonCodes: gates.forbidToolSelectionReasonCodes,
forbidSelectedToolSelectionReasonCodes: gates.forbidSelectedToolSelectionReasonCodes,
forbidOmittedToolSelectionReasonCodes: gates.forbidOmittedToolSelectionReasonCodes,
maxRecoveryCaseRuns,
failOnRecoveryIncrease,
requireStableRuntimeFingerprint: gates.requireStableRuntimeFingerprint,
requireStableRuntimeBounds: gates.requireStableRuntimeBounds,
};
const gateFailures = [];
if (history.found < history.requested) {
gateFailures.push(`expected at least ${history.requested} summary artifacts, found ${history.found}`);
}
for (const run of history.runs) {
if (run.parseError) {
gateFailures.push(`${run.runId}: summary parse error`);
continue;
}
if (run.ok !== true) {
gateFailures.push(`${run.runId}: expected ok=true, got ${run.ok}`);
}
if (run.failures > 0) {
gateFailures.push(`${run.runId}: expected failures=0, got ${run.failures}`);
}
if (run.debugSummaryStatus > 0) {
gateFailures.push(`${run.runId}: expected debug_summary_status=0, got ${run.debugSummaryStatus}`);
}
if (hardLimits.requireRunKinds !== undefined && !runKindMatches(run.runKind, hardLimits.requireRunKinds)) {
gateFailures.push(
`${run.runId}: expected run_kind=${formatRunKinds(hardLimits.requireRunKinds)}, got ${run.runKind || "(missing)"}`,
);
}
if (hardLimits.expectCases !== undefined && run.cases !== hardLimits.expectCases) {
gateFailures.push(`${run.runId}: expected cases=${hardLimits.expectCases}, got ${run.cases}`);
}
if (
hardLimits.expectCaseNames !== undefined &&
!caseNameListsEqual(run.selectedCases, hardLimits.expectCaseNames)
) {
gateFailures.push(
`${run.runId}: expected case_names=${formatCaseNames(hardLimits.expectCaseNames)}, got ${formatCaseNames(run.selectedCases)}`,
);
}
if (run.errors > hardLimits.maxErrors) {
gateFailures.push(`${run.runId}: expected errors<=${hardLimits.maxErrors}, got ${run.errors}`);
}
if (run.processLifecycleFailures > 0) {
gateFailures.push(`${run.runId}: expected process_lifecycle_failures=0, got ${run.processLifecycleFailures}`);
}
if (run.providerErrors > 0) {
gateFailures.push(`${run.runId}: expected provider_errors=0, got ${run.providerErrors}`);
}
if (run.emptyAssistantEnds > hardLimits.maxEmptyAssistantEnds) {
gateFailures.push(
`${run.runId}: expected empty_assistant_ends<=${hardLimits.maxEmptyAssistantEnds}, got ${run.emptyAssistantEnds}`,
);
}
if (run.rawToolMarkupFinalAnswers > hardLimits.maxRawToolMarkupFinalAnswers) {
gateFailures.push(
`${run.runId}: expected raw_tool_markup_final_answers<=${hardLimits.maxRawToolMarkupFinalAnswers}, got ${run.rawToolMarkupFinalAnswers}`,
);
}
if (run.toolEnvelopeFinalAnswers > hardLimits.maxToolEnvelopeFinalAnswers) {
gateFailures.push(
`${run.runId}: expected tool_envelope_final_answers<=${hardLimits.maxToolEnvelopeFinalAnswers}, got ${run.toolEnvelopeFinalAnswers}`,
);
}
if (hardLimits.maxRecoveries !== undefined && run.recoveries > hardLimits.maxRecoveries) {
gateFailures.push(`${run.runId}: expected recoveries<=${hardLimits.maxRecoveries}, got ${run.recoveries}`);
}
if (hardLimits.maxRecoveryRate !== undefined && run.recoveryRate > hardLimits.maxRecoveryRate) {
gateFailures.push(
`${run.runId}: expected recovery_rate<=${hardLimits.maxRecoveryRate}, got ${run.recoveryRate.toFixed(4)}`,
);
}
if (
hardLimits.maxRequestLatencyMs !== undefined &&
run.requestLatencyMsMax > hardLimits.maxRequestLatencyMs
) {
gateFailures.push(
`${run.runId}: expected request_latency_ms_max<=${hardLimits.maxRequestLatencyMs}, got ${run.requestLatencyMsMax}`,
);
}
if (hardLimits.maxSlowRequests !== undefined && run.slowRequestCount > hardLimits.maxSlowRequests) {
gateFailures.push(
`${run.runId}: expected slow_requests<=${hardLimits.maxSlowRequests}, got ${run.slowRequestCount}`,
);
}
gateFailures.push(...buildReasonCodeGateFailures(
run.runId,
"tool_selection_reason_codes",
run.toolSelectionReasonCodes,
hardLimits.requireToolSelectionReasonCodes,
hardLimits.forbidToolSelectionReasonCodes,
));
gateFailures.push(...buildReasonCodeGateFailures(
run.runId,
"selected_tool_selection_reason_codes",
run.selectedToolSelectionReasonCodes,
hardLimits.requireSelectedToolSelectionReasonCodes,
hardLimits.forbidSelectedToolSelectionReasonCodes,
));
gateFailures.push(...buildReasonCodeGateFailures(
run.runId,
"omitted_tool_selection_reason_codes",
run.omittedToolSelectionReasonCodes,
hardLimits.requireOmittedToolSelectionReasonCodes,
hardLimits.forbidOmittedToolSelectionReasonCodes,
));
}
const recoveryTrend = buildRecoveryTrend(history.runs);
if (
hardLimits.failOnRecoveryIncrease &&
recoveryTrend.previousRunId &&
(recoveryTrend.recoveryDelta > 0 || recoveryTrend.recoveryRateDelta > 0)
) {
gateFailures.push(
`recovery increased: ${recoveryTrend.previousRunId}->${recoveryTrend.latestRunId} ` +
`recoveries_delta=${recoveryTrend.recoveryDelta} recovery_rate_delta=${recoveryTrend.recoveryRateDelta.toFixed(4)}`,
);
}
const recoveryCaseRunCounts = buildRecoveryCaseRunCounts(history.runs);
const repeatedRecoveryCases = Object.entries(recoveryCaseRunCounts)
.filter(([, count]) => count > 1)
.map(([caseName, runCount]) => ({ caseName, runCount }));
if (hardLimits.maxRecoveryCaseRuns !== undefined) {
for (const [caseName, runCount] of Object.entries(recoveryCaseRunCounts)) {
if (runCount > hardLimits.maxRecoveryCaseRuns) {
gateFailures.push(
`${caseName}: expected recovery_case_runs<=${hardLimits.maxRecoveryCaseRuns}, got ${runCount}`,
);
}
}
}
const runtimeStability = buildRuntimeStability(history.runs);
if (
hardLimits.requireStableRuntimeFingerprint &&
runtimeStability.runtimeFingerprints.length > 1
) {
gateFailures.push(
`runtime_fingerprint changed across selected runs: ${formatSignatureGroups(runtimeStability.runtimeFingerprints)}`,
);
}
if (hardLimits.requireStableRuntimeBounds && runtimeStability.runtimeBounds.length > 1) {
gateFailures.push(
`runtime_bounds changed across selected runs: ${formatSignatureGroups(runtimeStability.runtimeBounds)}`,
);
}
return {
limits: hardLimits,
recoveryTrend,
recoveryCaseRunCounts,
repeatedRecoveryCases,
runtimeStability,
gateFailures,
};
}
function printTrendGate(limit) {
const history = buildHistory(limit);
const evaluation = evaluateTrendGate(history);
const trendGate = {
schema: "xtalpi-pi-tools.smoke-trend-gate.v1",
outDir,
requested: history.requested,
totalArtifacts: history.totalArtifacts,
candidateArtifacts: history.candidateArtifacts,
filteredOutArtifacts: history.filteredOutArtifacts,
filter: history.filter,
found: history.found,
order: history.order,
ok: evaluation.gateFailures.length === 0,
...evaluation,
history,
};
if (format === "json") {
console.log(JSON.stringify(trendGate, null, 2));
} else {
console.log("xtalpi-pi-tools smoke trend gate");
console.log(
`out_dir=${outDir} requested=${history.requested} found=${history.found} ` +
`total_artifacts=${history.totalArtifacts} candidate_artifacts=${history.candidateArtifacts} ` +
`filtered_out_artifacts=${history.filteredOutArtifacts} ` +
`run_kind_filter=${history.filter.runKinds ? history.filter.runKinds.join(",") : "(none)"} ` +
`order=${history.order} ok=${trendGate.ok}`,
);
console.log(
`latest=${evaluation.recoveryTrend.latestRunId || "(none)"} ` +
`previous=${evaluation.recoveryTrend.previousRunId || "(none)"} ` +
`recoveries_delta=${evaluation.recoveryTrend.recoveryDelta} ` +
`recovery_rate_delta=${evaluation.recoveryTrend.recoveryRateDelta.toFixed(4)}`,
);
if (evaluation.repeatedRecoveryCases.length > 0) {
console.log(`repeated_recovery_cases=${JSON.stringify(evaluation.repeatedRecoveryCases)}`);
}
if (
evaluation.limits.requireStableRuntimeFingerprint ||
evaluation.limits.requireStableRuntimeBounds ||
evaluation.runtimeStability.runtimeFingerprints.length > 1 ||
evaluation.runtimeStability.runtimeBounds.length > 1
) {
console.log(
`runtime_stability=${JSON.stringify({
runtimeFingerprints: evaluation.runtimeStability.runtimeFingerprints.map((item) => ({
sha256: item.sha256,
count: item.count,
runIds: item.runIds,
})),
runtimeBounds: evaluation.runtimeStability.runtimeBounds.map((item) => ({
sha256: item.sha256,
count: item.count,
runIds: item.runIds,
})),
})}`,
);
}
if (evaluation.gateFailures.length > 0) {
console.error(`gate_failures=${JSON.stringify(evaluation.gateFailures)}`);
}
}
process.exit(evaluation.gateFailures.length > 0 ? 1 : 0);
}
if (!fs.existsSync(outDir) || !fs.statSync(outDir).isDirectory()) {
console.error(`xtalpi-pi-tools debug summary: directory not found: ${outDir}`);
process.exit(1);
}
if (retentionReport) {
printRetentionReport();
process.exit(0);
}
if (historyLimit !== undefined) {
printHistory(historyLimit);
}
if (trendGateLimit !== undefined) {
printTrendGate(trendGateLimit);
}
if (driftLimit !== undefined) {
printDrift(driftLimit);
}
if (compareMode) {
printCompare(compareBaseRunId, compareHeadRunId);
}
let debugFiles = fs.readdirSync(outDir)
.filter((file) => file.endsWith(".debug.jsonl"))
.sort();
if (debugFiles.length === 0) {
console.error(`xtalpi-pi-tools debug summary: no *.debug.jsonl files found in ${outDir}`);
process.exit(1);
}
let selectedRunId = runIdFilter || undefined;
if (runIdFilter) {
debugFiles = debugFiles.filter((file) => file.startsWith(`${runIdFilter}-`));
} else if (latestOnly) {
const runIds = debugFiles
.map((file) => file.match(/^(\d{8}-\d{6}(?:-\d+)?)-/)?.[1])
.filter(Boolean)
.sort();
const latestRunId = runIds.at(-1);
if (latestRunId) {
selectedRunId = latestRunId;
debugFiles = debugFiles.filter((file) => file.startsWith(`${latestRunId}-`));
}
}
if (debugFiles.length === 0) {
const selector = runIdFilter ? `run id ${runIdFilter}` : "selection";
console.error(`xtalpi-pi-tools debug summary: no *.debug.jsonl files matched ${selector} in ${outDir}`);
process.exit(1);
}
const cases = debugFiles.map(summarizeCase);
const totals = {
cases: cases.length,
debugEvents: 0,
debugParseErrors: 0,
mainParseErrors: 0,
turns: 0,
toolCalls: 0,
recoveries: 0,
emptyAssistantEnds: 0,
rawToolMarkupFinalAnswers: 0,
toolEnvelopeFinalAnswers: 0,
piToolStarts: 0,
errors: 0,
lifecycleArtifacts: 0,
processLifecycleFailures: 0,
watchdogTimeouts: 0,
timedOutAfterAgentEnd: 0,
semanticFlowOkProcessFailures: 0,
postAgentEndLingerMaxSeconds: 0,
providerErrors: 0,
retryableProviderErrors: 0,
requestCount: 0,
requestLatencyMsMin: 0,
requestLatencyMsMax: 0,
requestLatencyMsAvg: 0,
slowRequestCount: 0,
slowRequestThresholdMs: SLOW_REQUEST_THRESHOLD_MS,
argumentValidationWarnings: 0,
toolSelectionClippedCases: 0,
toolSelectionOmittedCountMax: 0,
providerErrorCodes: {},
providerErrorCategories: {},
argumentValidationWarningCodes: {},
toolSelectionReasonCodes: {},
selectedToolSelectionReasonCodes: {},
omittedToolSelectionReasonCodes: {},
recoveryByEvent: {},
};
let requestLatencyMsTotal = 0;
for (const item of cases) {
totals.debugEvents += item.debugEvents;
totals.debugParseErrors += item.debugParseErrors;
totals.mainParseErrors += item.mainParseErrors;
totals.turns += item.turns;
totals.toolCalls += item.toolCalls;
totals.recoveries += item.recoveries;
totals.emptyAssistantEnds += item.emptyAssistantEnds;
totals.rawToolMarkupFinalAnswers += item.rawToolMarkupFinalAnswer ? 1 : 0;
totals.toolEnvelopeFinalAnswers += item.toolEnvelopeFinalAnswer ? 1 : 0;
totals.piToolStarts += item.piToolStarts.length;
totals.errors += item.errors;
totals.lifecycleArtifacts += item.lifecyclePresent ? 1 : 0;
totals.processLifecycleFailures += item.processLifecycleOk === false ? 1 : 0;
totals.watchdogTimeouts += item.timedOutByWatchdog ? 1 : 0;
totals.timedOutAfterAgentEnd += item.timedOutAfterAgentEnd ? 1 : 0;
totals.semanticFlowOkProcessFailures += item.semanticFlowOk && item.processLifecycleOk === false ? 1 : 0;
totals.providerErrors += item.providerErrors;
totals.retryableProviderErrors += item.retryableProviderErrors;
if (item.requestCount > 0) {
totals.requestLatencyMsMin = totals.requestCount === 0
? item.requestLatencyMsMin
: Math.min(totals.requestLatencyMsMin, item.requestLatencyMsMin);
totals.requestLatencyMsMax = Math.max(totals.requestLatencyMsMax, item.requestLatencyMsMax);
requestLatencyMsTotal += item.requestLatencyMsAvg * item.requestCount;
}
totals.requestCount += item.requestCount;
totals.slowRequestCount += item.slowRequestCount;
totals.argumentValidationWarnings += item.argumentValidationWarnings;
totals.toolSelectionClippedCases += item.toolSelectionClipped === true ? 1 : 0;
if (item.toolSelectionOmittedCountMax !== undefined) {
totals.toolSelectionOmittedCountMax = Math.max(totals.toolSelectionOmittedCountMax, item.toolSelectionOmittedCountMax);
}
if (typeof item.postAgentEndLingerSeconds === "number") {
totals.postAgentEndLingerMaxSeconds = Math.max(totals.postAgentEndLingerMaxSeconds, item.postAgentEndLingerSeconds);
}
for (const [code, count] of Object.entries(item.providerErrorCodes)) {
increment(totals.providerErrorCodes, code, count);
}
for (const [category, count] of Object.entries(item.providerErrorCategories)) {
increment(totals.providerErrorCategories, category, count);
}
for (const [code, count] of Object.entries(item.argumentValidationWarningCodes)) {
increment(totals.argumentValidationWarningCodes, code, count);
}
for (const [code, count] of Object.entries(item.toolSelectionReasonCodes)) {
increment(totals.toolSelectionReasonCodes, code, count);
}
for (const [code, count] of Object.entries(item.selectedToolSelectionReasonCodes)) {
increment(totals.selectedToolSelectionReasonCodes, code, count);
}
for (const [code, count] of Object.entries(item.omittedToolSelectionReasonCodes)) {
increment(totals.omittedToolSelectionReasonCodes, code, count);
}
for (const [event, count] of Object.entries(item.recoveryByEvent)) {
increment(totals.recoveryByEvent, event, count);
}
}
totals.recoveryRate = totals.turns > 0 ? totals.recoveries / totals.turns : 0;
totals.requestLatencyMsAvg = totals.requestCount > 0 ? Math.round(requestLatencyMsTotal / totals.requestCount) : 0;
const directCaseSet = buildCaseSet(cases.map((item) => item.caseName));
const directRunKind = classifyRunKind(directCaseSet);
const gateFailures = [];
if (gates.expectCases !== undefined && totals.cases !== gates.expectCases) {
gateFailures.push(`expected cases=${gates.expectCases}, got ${totals.cases}`);
}
if (gates.requireRunKinds !== undefined && !runKindMatches(directRunKind, gates.requireRunKinds)) {
gateFailures.push(`expected run_kind=${formatRunKinds(gates.requireRunKinds)}, got ${directRunKind}`);
}
if (
gates.expectCaseNames !== undefined &&
!caseNameListsEqual(cases.map((item) => item.caseName), gates.expectCaseNames)
) {
gateFailures.push(
`expected case_names=${formatCaseNames(gates.expectCaseNames)}, got ${formatCaseNames(cases.map((item) => item.caseName))}`,
);
}
if (gates.maxErrors !== undefined && totals.errors > gates.maxErrors) {
gateFailures.push(`expected errors<=${gates.maxErrors}, got ${totals.errors}`);
}
if (totals.debugParseErrors > 0 || totals.mainParseErrors > 0) {
gateFailures.push(`expected parse_errors=0, got debug=${totals.debugParseErrors} main=${totals.mainParseErrors}`);
}
if (totals.processLifecycleFailures > 0) {
gateFailures.push(`expected process_lifecycle_failures=0, got ${totals.processLifecycleFailures}`);
}
if (totals.providerErrors > 0) {
gateFailures.push(`expected provider_errors=0, got ${totals.providerErrors}`);
}
if (gates.maxEmptyAssistantEnds !== undefined && totals.emptyAssistantEnds > gates.maxEmptyAssistantEnds) {
gateFailures.push(`expected empty_assistant_ends<=${gates.maxEmptyAssistantEnds}, got ${totals.emptyAssistantEnds}`);
}
if (
gates.maxRawToolMarkupFinalAnswers !== undefined &&
totals.rawToolMarkupFinalAnswers > gates.maxRawToolMarkupFinalAnswers
) {
gateFailures.push(
`expected raw_tool_markup_final_answers<=${gates.maxRawToolMarkupFinalAnswers}, got ${totals.rawToolMarkupFinalAnswers}`,
);
}
if (gates.maxRecoveries !== undefined && totals.recoveries > gates.maxRecoveries) {
gateFailures.push(`expected recoveries<=${gates.maxRecoveries}, got ${totals.recoveries}`);
}
if (gates.maxRecoveryRate !== undefined && totals.recoveryRate > gates.maxRecoveryRate) {
gateFailures.push(`expected recovery_rate<=${gates.maxRecoveryRate}, got ${totals.recoveryRate.toFixed(4)}`);
}
if (gates.maxRequestLatencyMs !== undefined && totals.requestLatencyMsMax > gates.maxRequestLatencyMs) {
gateFailures.push(
`expected request_latency_ms_max<=${gates.maxRequestLatencyMs}, got ${totals.requestLatencyMsMax}`,
);
}
if (gates.maxSlowRequests !== undefined && totals.slowRequestCount > gates.maxSlowRequests) {
gateFailures.push(`expected slow_requests<=${gates.maxSlowRequests}, got ${totals.slowRequestCount}`);
}
gateFailures.push(...buildReasonCodeGateFailures(
"totals",
"tool_selection_reason_codes",
totals.toolSelectionReasonCodes,
gates.requireToolSelectionReasonCodes,
gates.forbidToolSelectionReasonCodes,
));
gateFailures.push(...buildReasonCodeGateFailures(
"totals",
"selected_tool_selection_reason_codes",
totals.selectedToolSelectionReasonCodes,
gates.requireSelectedToolSelectionReasonCodes,
gates.forbidSelectedToolSelectionReasonCodes,
));
gateFailures.push(...buildReasonCodeGateFailures(
"totals",
"omitted_tool_selection_reason_codes",
totals.omittedToolSelectionReasonCodes,
gates.requireOmittedToolSelectionReasonCodes,
gates.forbidOmittedToolSelectionReasonCodes,
));
const summary = { outDir, latestOnly, runId: selectedRunId, caseSet: directCaseSet, runKind: directRunKind, gates, gateFailures, totals, cases };
if (format === "json") {
console.log(JSON.stringify(summary, null, 2));
} else {
console.log("xtalpi-pi-tools debug summary");
console.log(`out_dir=${outDir} latest_only=${latestOnly} run_id=${selectedRunId || "(all)"}`);
console.log(
`cases=${totals.cases} run_kind=${directRunKind} debug_events=${totals.debugEvents} turns=${totals.turns} ` +
`tool_calls=${totals.toolCalls} recoveries=${totals.recoveries} recovery_rate=${totals.recoveryRate.toFixed(4)} ` +
`empty_assistant_ends=${totals.emptyAssistantEnds} raw_tool_markup_final_answers=${totals.rawToolMarkupFinalAnswers} ` +
`tool_envelope_final_answers=${totals.toolEnvelopeFinalAnswers} ` +
`pi_tool_starts=${totals.piToolStarts} errors=${totals.errors} ` +
`lifecycle_artifacts=${totals.lifecycleArtifacts} process_lifecycle_failures=${totals.processLifecycleFailures} ` +
`watchdog_timeouts=${totals.watchdogTimeouts} timed_out_after_agent_end=${totals.timedOutAfterAgentEnd} ` +
`semantic_flow_ok_process_failures=${totals.semanticFlowOkProcessFailures} ` +
`post_agent_end_linger_max_seconds=${totals.postAgentEndLingerMaxSeconds} ` +
`provider_errors=${totals.providerErrors} retryable_provider_errors=${totals.retryableProviderErrors} ` +
`request_latency_ms=${totals.requestLatencyMsMax}/${totals.requestLatencyMsAvg}/${totals.requestCount} ` +
`slow_requests=${totals.slowRequestCount} slow_request_threshold_ms=${totals.slowRequestThresholdMs} ` +
`argument_validation_warnings=${totals.argumentValidationWarnings} ` +
`tool_selection_clipped_cases=${totals.toolSelectionClippedCases} ` +
`tool_selection_omitted_count_max=${totals.toolSelectionOmittedCountMax}`,
);
if (Object.keys(totals.providerErrorCodes).length > 0) {
console.log(`provider_error_codes=${JSON.stringify(totals.providerErrorCodes)}`);
}
if (Object.keys(totals.providerErrorCategories).length > 0) {
console.log(`provider_error_categories=${JSON.stringify(totals.providerErrorCategories)}`);
}
if (Object.keys(totals.argumentValidationWarningCodes).length > 0) {
console.log(`argument_validation_warning_codes=${JSON.stringify(totals.argumentValidationWarningCodes)}`);
}
if (Object.keys(totals.toolSelectionReasonCodes).length > 0) {
console.log(`tool_selection_reason_codes=${JSON.stringify(totals.toolSelectionReasonCodes)}`);
}
if (Object.keys(totals.selectedToolSelectionReasonCodes).length > 0) {
console.log(`selected_tool_selection_reason_codes=${JSON.stringify(totals.selectedToolSelectionReasonCodes)}`);
}
if (Object.keys(totals.omittedToolSelectionReasonCodes).length > 0) {
console.log(`omitted_tool_selection_reason_codes=${JSON.stringify(totals.omittedToolSelectionReasonCodes)}`);
}
if (Object.keys(totals.recoveryByEvent).length > 0) {
console.log(`recovery_by_event=${JSON.stringify(totals.recoveryByEvent)}`);
}
for (const item of cases) {
const recoveryText = Object.keys(item.recoveryByEvent).length > 0
? ` recovery_by_event=${JSON.stringify(item.recoveryByEvent)}`
: "";
const toolText = item.piToolStarts.length > 0 ? ` pi_tools=${item.piToolStarts.join(",")}` : "";
const selectedText = item.selectedToolCountMax !== undefined
? ` selected_tools=${item.selectedToolCountMin}-${item.selectedToolCountMax}`
: "";
const selectionText = item.toolSelectionClipped === true || item.toolSelectionOmittedCountMax !== undefined
? ` tool_selection_clipped=${item.toolSelectionClipped}` +
` tool_selection_omitted=${item.toolSelectionOmittedCountMin ?? "(unknown)"}-${item.toolSelectionOmittedCountMax ?? "(unknown)"}` +
` tool_selection_valid=${item.toolSelectionValidCountMin ?? "(unknown)"}-${item.toolSelectionValidCountMax ?? "(unknown)"}`
: "";
const toolSelectionPromptSources = item.runtimeFingerprint?.toolSelectionPromptSources || [];
const promptSourceText = toolSelectionPromptSources.length
? ` tool_selection_prompt_source=${toolSelectionPromptSources.join("|")}`
: "";
const providerErrorText = item.providerErrors > 0
? ` provider_errors=${item.providerErrors}` +
` retryable_provider_errors=${item.retryableProviderErrors}` +
` provider_error_codes=${JSON.stringify(item.providerErrorCodes)}` +
` provider_error_categories=${JSON.stringify(item.providerErrorCategories)}` +
` provider_http_statuses=${item.providerHttpStatuses.join(",") || "(none)"}`
: "";
const argumentWarningText = item.argumentValidationWarnings > 0
? ` argument_validation_warnings=${item.argumentValidationWarnings}` +
` argument_validation_warning_codes=${JSON.stringify(item.argumentValidationWarningCodes)}`
: "";
const toolSelectionReasonText = Object.keys(item.toolSelectionReasonCodes).length > 0
? ` tool_selection_reason_codes=${JSON.stringify(item.toolSelectionReasonCodes)}` +
` selected_tool_selection_reason_codes=${JSON.stringify(item.selectedToolSelectionReasonCodes)}` +
` omitted_tool_selection_reason_codes=${JSON.stringify(item.omittedToolSelectionReasonCodes)}`
: "";
const requestLatencyText =
` request_latency_ms=${item.requestLatencyMsMax}/${item.requestLatencyMsAvg}/${item.requestCount}` +
` slow_requests=${item.slowRequestCount}` +
` slow_request_threshold_ms=${item.slowRequestThresholdMs}`;
const lifecycleText = item.lifecyclePresent
? ` process_lifecycle_ok=${item.processLifecycleOk}` +
` timed_out_by_watchdog=${item.timedOutByWatchdog}` +
` timed_out_after_agent_end=${item.timedOutAfterAgentEnd}` +
` semantic_flow_ok=${item.semanticFlowOk}` +
` elapsed_seconds=${item.elapsedSeconds ?? "(unknown)"}` +
` post_agent_end_linger_seconds=${item.postAgentEndLingerSeconds ?? "(unknown)"}`
: "";
console.log(
`- ${item.runId}/${item.caseName}: debug_events=${item.debugEvents} turns=${item.turns} tool_calls=${item.toolCalls}` +
` recoveries=${item.recoveries} empty_assistant_ends=${item.emptyAssistantEnds}` +
` raw_tool_markup_final_answer=${item.rawToolMarkupFinalAnswer}` +
` tool_envelope_final_answer=${item.toolEnvelopeFinalAnswer}${recoveryText}${toolText}${selectedText}${selectionText}` +
`${promptSourceText}${providerErrorText}${argumentWarningText}${toolSelectionReasonText}${requestLatencyText}${lifecycleText}` +
` final_text_chars=${item.finalTextChars}`,
);
}
if (gateFailures.length > 0) {
console.error(`gate_failures=${JSON.stringify(gateFailures)}`);
}
}
process.exit(gateFailures.length > 0 ? 1 : 0);
NODE