/** * Canonical incident-drill catalog for the Play Runtime. * * A drill is not a retry policy and it never changes a Play by itself. It * states a bounded fault we can deliberately exercise, the topology where it * is meaningful, and the durable safety facts an observer must verify. Test * seams, the local admin CLI, V2 scenarios, and documentation are Adapters to * this Module; they must not invent their own outcome descriptions. */ export const RUNTIME_TEST_FAULT_IDS = [ 'receipt_complete_write_fail', 'receipt_fail_write_fail', 'worker_receipt_complete_write_fail', 'invocation_response_delivery_abort', 'receipt_gateway_hold_ms', 'receipt_claim_query_hold_once_ms', 'receipt_claim_response_timeout', 'modal_sandbox_create_resource_exhausted', 'runtime_sheet_page_tail_hold_ms', ] as const; export type RuntimeTestFaultName = (typeof RUNTIME_TEST_FAULT_IDS)[number]; export type RuntimeIncidentDrillId = | 'local_worker_process_outage' | 'local_gateway_process_outage' | 'local_runtime_process_outage' | RuntimeTestFaultName; export type RuntimeIncidentTopology = | 'local_daytona' | 'preview_fly' | 'preview_direct_modal'; export type RuntimeIncidentSafetyFact = | 'same_logical_run' | 'no_false_success' | 'bounded_provider_execution' | 'retry_or_explicit_recovery' | 'no_user_code_before_recovery' | 'all_rows_settle_once'; /** * The automated evidence that proves this drill. A focused regression owns a * narrow storage/protocol invariant; a black-box scenario drives the full * local (or Preview) runtime. Keeping this on the catalog means on-call and * CI see the same answer to “how do we prove this?” as the injector. */ export type RuntimeIncidentVerification = Readonly<{ kind: 'focused_regression' | 'local_black_box' | 'preview_black_box'; target: string; }>; export type RuntimeIncidentDrill = Readonly<{ id: RuntimeIncidentDrillId; description: string; topologies: readonly RuntimeIncidentTopology[]; control: | Readonly<{ kind: 'process'; component: 'worker' | 'gateway' | 'all'; modes: readonly ('pause' | 'crash')[]; }> | Readonly<{ kind: 'runtime_test_fault'; defaultValue: number; valueMeaning: 'occurrence' | 'milliseconds'; }>; expected: Readonly<{ outcome: string; safetyFacts: readonly RuntimeIncidentSafetyFact[]; }>; verification: RuntimeIncidentVerification; }>; const LOCAL_TOPOLOGY = ['local_daytona'] as const; const LOCAL_AND_PREVIEW_TOPOLOGIES = [ 'local_daytona', 'preview_fly', 'preview_direct_modal', ] as const; const MODAL_PREVIEW_TOPOLOGY = ['preview_direct_modal'] as const; /** * The single catalog that binds an injected fault to its required proof. * * `valueMeaning` is deliberately explicit: callers cannot accidentally pass * a millisecond delay where the runtime interprets an occurrence count. */ export const RUNTIME_INCIDENT_DRILL_CATALOG: readonly RuntimeIncidentDrill[] = [ { id: 'local_worker_process_outage', description: 'Interrupt the scheduler worker in the selected non-production runtime topology.', topologies: LOCAL_AND_PREVIEW_TOPOLOGIES, control: { kind: 'process', component: 'worker', modes: ['pause', 'crash'], }, expected: { outcome: 'The same logical run stays durable through the worker outage and completes after the selected topology recovers.', safetyFacts: ['same_logical_run', 'no_false_success'], }, verification: { kind: 'local_black_box', target: 'local-component-outage-recovery-e2e', }, }, { id: 'local_gateway_process_outage', description: 'Interrupt the receipt gateway in the selected non-production runtime topology.', topologies: LOCAL_AND_PREVIEW_TOPOLOGIES, control: { kind: 'process', component: 'gateway', modes: ['pause', 'crash'], }, expected: { outcome: 'In-flight work recovers only at a safe receipt boundary and the same logical run completes after the gateway recovers.', safetyFacts: [ 'same_logical_run', 'no_false_success', 'bounded_provider_execution', ], }, verification: { kind: 'local_black_box', target: 'local-component-outage-recovery-e2e', }, }, { id: 'local_runtime_process_outage', description: 'Interrupt both runtime components in the selected non-production topology.', topologies: LOCAL_AND_PREVIEW_TOPOLOGIES, control: { kind: 'process', component: 'all', modes: ['pause', 'crash'] }, expected: { outcome: 'Durable work waits through the combined outage; it never becomes a false success or creates a second logical run.', safetyFacts: ['same_logical_run', 'no_false_success'], }, verification: { kind: 'local_black_box', target: 'local-component-outage-recovery-e2e', }, }, { id: 'receipt_complete_write_fail', description: 'Fail the next durable receipt-completion write.', topologies: LOCAL_AND_PREVIEW_TOPOLOGIES, control: { kind: 'runtime_test_fault', defaultValue: 1, valueMeaning: 'occurrence', }, expected: { outcome: 'The failed run is loud, and a later bounded recovery may repeat the affected provider operation if its prior completion was not durable.', safetyFacts: ['retry_or_explicit_recovery', 'bounded_provider_execution'], }, verification: { kind: 'local_black_box', target: 'receipt-persist-failure-run-fatal-e2e', }, }, { id: 'receipt_fail_write_fail', description: 'Fail the next durable receipt-failure write.', topologies: LOCAL_AND_PREVIEW_TOPOLOGIES, control: { kind: 'runtime_test_fault', defaultValue: 1, valueMeaning: 'occurrence', }, expected: { outcome: 'The failure is loud; recovery remains bounded and may repeat an operation whose prior outcome was not durably recorded.', safetyFacts: ['retry_or_explicit_recovery', 'bounded_provider_execution'], }, verification: { kind: 'focused_regression', target: 'tests/lib/plays/runtime-test-fault-registry.test.ts', }, }, { id: 'worker_receipt_complete_write_fail', description: 'Fail the worker-side receipt-completion write once.', topologies: LOCAL_AND_PREVIEW_TOPOLOGIES, control: { kind: 'runtime_test_fault', defaultValue: 1, valueMeaning: 'occurrence', }, expected: { outcome: 'The worker reports the persistence failure loudly; bounded recovery may repeat work whose completion was not durable.', safetyFacts: ['retry_or_explicit_recovery', 'bounded_provider_execution'], }, verification: { kind: 'focused_regression', target: 'tests/lib/plays/runtime-test-fault-registry.test.ts', }, }, { id: 'invocation_response_delivery_abort', description: 'Abort one receipt-gateway response after invocation delivery.', topologies: LOCAL_AND_PREVIEW_TOPOLOGIES, control: { kind: 'runtime_test_fault', defaultValue: 1, valueMeaning: 'occurrence', }, expected: { outcome: 'The same logical run completes through receipt replay; recovery remains bounded if the provider outcome must be retried.', safetyFacts: ['same_logical_run', 'bounded_provider_execution'], }, verification: { kind: 'local_black_box', target: 'invocation-delivery-replay-e2e', }, }, { id: 'receipt_gateway_hold_ms', description: 'Hold one checked-out receipt-gateway scheduler client.', topologies: LOCAL_AND_PREVIEW_TOPOLOGIES, control: { kind: 'runtime_test_fault', defaultValue: 500, valueMeaning: 'milliseconds', }, expected: { outcome: 'The request is delayed inside the real gateway pool and either completes within its deadline or follows bounded transport recovery.', safetyFacts: ['bounded_provider_execution'], }, verification: { kind: 'local_black_box', target: 'agent-heavy-gateway-load-e2e', }, }, { id: 'receipt_claim_query_hold_once_ms', description: 'Hold one physical receipt-claim query.', topologies: LOCAL_AND_PREVIEW_TOPOLOGIES, control: { kind: 'runtime_test_fault', defaultValue: 500, valueMeaning: 'milliseconds', }, expected: { outcome: 'The checked-out claim either completes or times out through bounded recovery; an ambiguous outcome may be retried within its budget.', safetyFacts: ['bounded_provider_execution'], }, verification: { kind: 'local_black_box', target: 'receipt-claim-deadline-recovery-e2e', }, }, { id: 'receipt_claim_response_timeout', description: 'Report one fully received receipt-claim response as timed out.', topologies: LOCAL_AND_PREVIEW_TOPOLOGIES, control: { kind: 'runtime_test_fault', defaultValue: 1, valueMeaning: 'occurrence', }, expected: { outcome: 'The same logical run completes by replaying the committed receipt response or uses bounded recovery if that response is unavailable.', safetyFacts: ['same_logical_run', 'bounded_provider_execution'], }, verification: { kind: 'local_black_box', target: 'receipt-ambiguous-response-recovery-e2e', }, }, { id: 'modal_sandbox_create_resource_exhausted', description: 'Make Modal sandbox creation return RESOURCE_EXHAUSTED before user code.', topologies: MODAL_PREVIEW_TOPOLOGY, control: { kind: 'runtime_test_fault', defaultValue: 1, valueMeaning: 'occurrence', }, expected: { outcome: 'The same logical run enters bounded pre-code defer/recovery and later retries sandbox startup; no user code or provider call precedes the retry.', safetyFacts: [ 'same_logical_run', 'no_user_code_before_recovery', 'bounded_provider_execution', ], }, verification: { kind: 'preview_black_box', target: 'modal-sandbox-recovery-e2e', }, }, { id: 'runtime_sheet_page_tail_hold_ms', description: 'Hold one runtime-sheet tail-page operation.', topologies: LOCAL_TOPOLOGY, control: { kind: 'runtime_test_fault', defaultValue: 500, valueMeaning: 'milliseconds', }, expected: { outcome: 'Page-tail work stays pending during the bounded hold, then completes without losing or duplicating rows.', safetyFacts: ['all_rows_settle_once'], }, verification: { kind: 'local_black_box', target: 'page-tail-sheet-write-hold-e2e', }, }, ]; export type RuntimeTestFaultDrill = Omit< RuntimeIncidentDrill, 'id' | 'control' > & Readonly<{ id: RuntimeTestFaultName; control: Readonly<{ kind: 'runtime_test_fault'; defaultValue: number; valueMeaning: 'occurrence' | 'milliseconds'; }>; }>; function isRuntimeTestFaultDrill( drill: RuntimeIncidentDrill, ): drill is RuntimeTestFaultDrill { return ( drill.control.kind === 'runtime_test_fault' && RUNTIME_TEST_FAULT_IDS.some((faultId) => faultId === drill.id) ); } export function runtimeTestFaultDrills(): readonly RuntimeTestFaultDrill[] { return RUNTIME_INCIDENT_DRILL_CATALOG.filter(isRuntimeTestFaultDrill); } export function findRuntimeIncidentDrill( id: string, ): RuntimeIncidentDrill | null { return ( RUNTIME_INCIDENT_DRILL_CATALOG.find((drill) => drill.id === id) ?? null ); } export function findRuntimeTestFaultDrill( id: string, ): RuntimeTestFaultDrill | null { const drill = findRuntimeIncidentDrill(id); return drill && isRuntimeTestFaultDrill(drill) ? drill : null; } export function runtimeIncidentDrillSupportsTopology( drill: RuntimeIncidentDrill, topology: RuntimeIncidentTopology, ): boolean { return drill.topologies.includes(topology); }