import type { AssertionResult, JsonValue, NodeConfigBase, NodeId, RunEvent, RunId, TestCaseRunStatus, TestSuiteRunStatus, WorkflowDefinition, } from "@codemation/core"; import { deriveAssertionPassed } from "@codemation/core"; import type { TestSuiteRunResult } from "@codemation/core/bootstrap"; import type { TestAssertionRepository } from "../../domain/runs/TestAssertionRepository"; import type { TestSuiteRunRepository } from "../../domain/runs/TestSuiteRunRepository"; import type { WorkflowRunRepository } from "../../domain/runs/WorkflowRunRepository"; import type { AssertionResultGuard } from "./AssertionResultGuard"; import type { TestAssertionIdFactory } from "./TestAssertionIdFactory"; export interface TestSuiteRunTrackerArgs { readonly workflow: WorkflowDefinition; readonly assertionIdFactory: TestAssertionIdFactory; readonly assertionRepo: TestAssertionRepository; readonly suiteRepo: TestSuiteRunRepository; readonly runRepo: WorkflowRunRepository; readonly assertionResultGuard: AssertionResultGuard; } export class TestSuiteRunTracker { private adoptedId: string | undefined; private readonly testRunCaseIndex = new Map(); private readonly nodeCoverage = new Set(); private readonly pendingEvents: RunEvent[] = []; private readonly pendingByRunId = new Map(); private readonly failedAssertionsByRunId = new Map(); private processingTail: Promise = Promise.resolve(); constructor(private readonly args: TestSuiteRunTrackerArgs) {} adopt(testSuiteRunId: string): void { this.adoptedId = testSuiteRunId; const queued = this.pendingEvents.splice(0, this.pendingEvents.length); for (const event of queued) { void this.onEvent(event); } } onEvent(event: RunEvent): Promise { const next = this.processingTail.then(async () => { await this.processEvent(event); }); this.processingTail = next.catch(() => undefined); return next; } private async processEvent(event: RunEvent): Promise { if (this.adoptedId === undefined) { this.pendingEvents.push(event); return; } switch (event.kind) { case "testCaseStarted": if (event.testSuiteRunId !== this.adoptedId) return; this.testRunCaseIndex.set(event.runId, event.testCaseIndex); this.failedAssertionsByRunId.set(event.runId, false); await this.persistCaseStarted(event); await this.drainPendingForRun(event.runId); return; case "testCaseCompleted": if (event.testSuiteRunId !== this.adoptedId) return; await this.drainPendingForRun(event.runId); await this.persistCaseCompleted(event); this.testRunCaseIndex.delete(event.runId); this.pendingByRunId.delete(event.runId); return; case "nodeCompleted": if (!this.testRunCaseIndex.has(event.runId)) { const queued = this.pendingByRunId.get(event.runId) ?? []; queued.push(event); this.pendingByRunId.set(event.runId, queued); return; } this.nodeCoverage.add(event.snapshot.nodeId); await this.persistAssertionsForCompletedNode(event); return; default: return; } } async finalize(orchestratorResult: TestSuiteRunResult): Promise { if (this.adoptedId === undefined) return; await this.processingTail; const childRuns = await this.args.suiteRepo.listChildRuns(this.adoptedId); let passedCases: number; let failedCases: number; if (childRuns.length > 0) { passedCases = childRuns.filter((r) => r.testCaseStatus === "succeeded").length; failedCases = childRuns.filter((r) => r.testCaseStatus === "failed").length; } else { const failedFromAssertions = [...this.failedAssertionsByRunId.values()].filter(Boolean).length; failedCases = Math.max(orchestratorResult.failedCases, failedFromAssertions); passedCases = Math.max(0, orchestratorResult.totalCases - failedCases); } const status: TestSuiteRunStatus = orchestratorResult.status === "errored" || orchestratorResult.status === "cancelled" ? orchestratorResult.status : this.deriveSuiteStatusFromCounts(orchestratorResult.totalCases, passedCases, failedCases); await this.args.suiteRepo.update(this.adoptedId, { status, finishedAt: new Date().toISOString(), totalCases: orchestratorResult.totalCases, passedCases, failedCases, nodeCoverage: [...this.nodeCoverage], }); } private async drainPendingForRun(runId: RunId): Promise { const queued = this.pendingByRunId.get(runId); if (!queued || queued.length === 0) return; this.pendingByRunId.delete(runId); for (const event of queued) { if (event.kind === "nodeCompleted") { this.nodeCoverage.add(event.snapshot.nodeId); await this.persistAssertionsForCompletedNode(event); } } } private async persistCaseStarted(event: Extract): Promise { if (this.adoptedId === undefined) return; if (!this.args.runRepo.updateTestCaseStatus) return; try { await this.args.runRepo.updateTestCaseStatus(event.runId, "running"); } catch {} } private async persistCaseCompleted(event: Extract): Promise { if (this.adoptedId === undefined) return; if (!this.args.runRepo.updateTestCaseStatus) return; let finalStatus: TestCaseRunStatus = event.status; if (event.status === "succeeded" && this.failedAssertionsByRunId.get(event.runId)) { finalStatus = "failed"; } await this.args.runRepo.updateTestCaseStatus(event.runId, finalStatus); } private async persistAssertionsForCompletedNode(event: Extract): Promise { if (this.adoptedId === undefined) return; const nodeDef = this.args.workflow.nodes.find((n) => n.id === event.snapshot.nodeId); if (!nodeDef) return; const config = nodeDef.config as NodeConfigBase; if (config.emitsAssertions !== true) return; const items = event.snapshot.outputs?.main ?? []; if (items.length === 0) return; if (!this.testRunCaseIndex.has(event.runId)) return; for (const item of items) { const result = item.json as AssertionResult | undefined; if (!this.args.assertionResultGuard.isAssertionResult(result)) { continue; } if (!deriveAssertionPassed(result)) { this.failedAssertionsByRunId.set(event.runId, true); } await this.args.assertionRepo.record({ id: this.args.assertionIdFactory.makeAssertionId(), runId: event.runId, testSuiteRunId: this.adoptedId, workflowId: event.workflowId, nodeId: event.snapshot.nodeId, name: result.name, score: result.score, ...(result.passThreshold !== undefined ? { passThreshold: result.passThreshold } : {}), ...(result.errored === true ? { errored: true as const } : {}), ...(result.expected !== undefined ? { expected: result.expected as JsonValue } : {}), ...(result.actual !== undefined ? { actual: result.actual as JsonValue } : {}), ...(result.message !== undefined ? { message: result.message } : {}), ...(result.details !== undefined ? { details: result.details } : {}), createdAt: event.at, }); } } private deriveSuiteStatusFromCounts( totalCases: number, passedCases: number, failedCases: number, ): TestSuiteRunStatus { if (totalCases === 0) return "succeeded"; if (failedCases === 0) return "succeeded"; if (passedCases === 0) return "failed"; return "partial"; } }