import { getRuntimeEnv } from '../runtime-env'; import type { PlayRunnerEvent } from './protocol'; import { runtimeTestFaultDrills, type RuntimeTestFaultName, } from './runtime-incident-drills'; export const PLAY_RUNTIME_TEST_FAULT_HEADER = 'x-deepline-test-fault'; export type RuntimeTestFaultRegistry = { consume(name: RuntimeTestFaultName): boolean; remaining(): Record; }; type ParsedRuntimeTestFaults = | { ok: true; registry: RuntimeTestFaultRegistry | null } | { ok: false; status: 400 | 403; error: string }; type RuntimeTestSeamContext = { internalTokenHeader?: string | null; verifiedSyntheticExecutor?: boolean; }; type ParsedRuntimeTestFaultCounts = | { ok: true; counts: Record | null; headerValue: string | null; } | { ok: false; status: 400 | 403; error: string }; type ValidatedRuntimeTestFaultHeader = | { ok: true; headerValue: string | null } | { ok: false; status: 400 | 403; error: string }; export type RuntimeTestPolicyOverrides = { /** Opt-in bounded runner map latency profile for local/preview diagnosis. */ mapLatencyProfile?: boolean; /** Exercise provider pacing during fixture runs without dispatching provider traffic. */ enforceFixtureProviderPacing?: boolean; receiptLeaseTtlMs?: number; sheetAttemptLeaseMs?: number; heartbeatIntervalMs?: number; workBudgetYieldLimits?: Partial< Record< | 'elapsed' | 'subrequest' | 'provider' | 'tool' | 'receipt' | 'sheet' | 'log' | 'egress', number > >; batchGraceMs?: number; }; type ValidatedRuntimeTestPolicyOverrides = | { ok: true; overrides: RuntimeTestPolicyOverrides | null } | { ok: false; status: 400 | 403; error: string }; const SUPPORTED_RUNTIME_TEST_FAULTS = new Set( runtimeTestFaultDrills().map(({ id }) => id), ); const RUNTIME_TEST_POLICY_MS_FIELDS = new Set([ 'receiptLeaseTtlMs', 'sheetAttemptLeaseMs', 'heartbeatIntervalMs', 'batchGraceMs', ]); const RUNTIME_TEST_POLICY_WORK_BUDGET_KEYS = new Set([ 'elapsed', 'subrequest', 'provider', 'tool', 'receipt', 'sheet', 'log', 'egress', ]); const MAX_RUNTIME_TEST_POLICY_MS = 10 * 60_000; const MAX_RUNTIME_TEST_POLICY_YIELD_LIMIT = 1_000_000; function testRuntimeSeamsEnabled(): boolean { return process.env.DEEPLINE_TEST_RUNTIME_SEAMS === '1'; } function internalTokenAllowsRuntimeSeams( context: RuntimeTestSeamContext | undefined, ): boolean { const deeplineEnv = process.env.DEEPLINE_ENV?.trim().toLowerCase(); if (deeplineEnv === 'prod') return false; if (deeplineEnv === 'production') return false; const expected = process.env.DEEPLINE_INTERNAL_TOKEN?.trim(); const actual = context?.internalTokenHeader?.trim(); return Boolean(expected && actual && expected === actual); } function runtimeTestSeamsAuthorized( context: RuntimeTestSeamContext | undefined, ): boolean { return ( testRuntimeSeamsEnabled() || internalTokenAllowsRuntimeSeams(context) || context?.verifiedSyntheticExecutor === true ); } function parseRuntimeTestFaultHeader( rawHeader: string, ): Map | { error: string } { const faults = new Map(); for (const rawPart of rawHeader.split(',')) { const part = rawPart.trim(); if (!part) continue; const [rawName, rawCount] = part.split(':'); const name = rawName?.trim() as RuntimeTestFaultName | undefined; if (!name || !SUPPORTED_RUNTIME_TEST_FAULTS.has(name)) { return { error: `Unsupported runtime test fault "${rawName ?? part}".` }; } const count = rawCount === undefined || rawCount.trim() === '' ? 1 : Number.parseInt(rawCount.trim(), 10); if (!Number.isInteger(count) || count < 1) { return { error: `Runtime test fault "${name}" count must be a positive integer.`, }; } faults.set(name, (faults.get(name) ?? 0) + count); } return faults; } /** Parse a known fault count without authorizing a request. Unknown/skewed headers return 0. */ export function recognizedRuntimeTestFaultCount( rawHeader: string | null | undefined, name: RuntimeTestFaultName, ): number { const header = rawHeader?.trim(); if (!header) return 0; const parsed = parseRuntimeTestFaultHeader(header); if ('error' in parsed) return 0; return parsed.get(name) ?? 0; } /** * A provider-create fault cannot use process-local "already consumed" state: * a deferred Play Run normally resumes on another worker. Instead, the * persisted scheduler attempt is its consumption fence. `:1` means inject on * generation one only; `:N` intentionally selects generation N for diagnosis. * * The header itself is admitted only through `parseRuntimeTestFaultCounts` at * run admission. This second production guard ensures a persisted historical * test header can never synthesize a provider failure in a live runtime. */ export function shouldInjectModalSandboxCreateResourceExhausted(input: { runtimeTestFaultHeader: string | null | undefined; runAttempt: number | null | undefined; }): boolean { if (getRuntimeEnv() === 'prod') return false; const requestedAttempt = recognizedRuntimeTestFaultCount( input.runtimeTestFaultHeader, 'modal_sandbox_create_resource_exhausted', ); if (requestedAttempt < 1) return false; const runAttempt = input.runAttempt; return ( typeof runAttempt === 'number' && Number.isSafeInteger(runAttempt) && runAttempt >= 1 && runAttempt === requestedAttempt ); } export function runtimeSheetPageTailRowsContainTarget( rows: readonly unknown[], ): boolean { return rows.some((row) => { if (!row || typeof row !== 'object' || Array.isArray(row)) return false; const record = row as Record; return record.inputIndex === 999 || record.input_index === 999; }); } export function runtimeSheetPageTailWriteMarkerEvent( runtimeTestFaultHeader: string | null | undefined, rows: readonly Record[], tableNamespace: string, dbSessionStrategy: string | null | undefined, ): PlayRunnerEvent | null { if ( dbSessionStrategy !== 'gateway_only' || recognizedRuntimeTestFaultCount( runtimeTestFaultHeader, 'runtime_sheet_page_tail_hold_ms', ) <= 0 || !runtimeSheetPageTailRowsContainTarget(rows) ) { return null; } return { type: 'log', at: new Date().toISOString(), source: 'play', line: `[runtime.sheet-page-tail-write] phase=start table=${tableNamespace} rows=${rows.length}`, }; } export function runtimeSheetPageTailHoldCompletedEvent( holdMs: number | null | undefined, tableNamespace: string, ): PlayRunnerEvent | null { if (!Number.isInteger(holdMs) || (holdMs ?? 0) <= 0) return null; return { type: 'log', at: new Date().toISOString(), source: 'play', line: `[runtime.sheet-page-tail-hold] phase=finish table=${tableNamespace} hold_ms=${holdMs}`, }; } function readPositiveIntegerField(input: { value: unknown; path: string; max: number; }): number | { error: string } { if ( typeof input.value !== 'number' || !Number.isFinite(input.value) || !Number.isInteger(input.value) || input.value < 1 || input.value > input.max ) { return { error: `${input.path} must be an integer between 1 and ${input.max}.`, }; } return input.value; } function parseRuntimeTestPolicyOverrides( value: unknown, ): RuntimeTestPolicyOverrides | { error: string } | null { if (value === undefined || value === null) { return null; } if (typeof value !== 'object' || Array.isArray(value)) { return { error: 'testPolicyOverrides must be a JSON object.' }; } const record = value as Record; const unknownKeys = Object.keys(record).filter( (key) => !RUNTIME_TEST_POLICY_MS_FIELDS.has(key) && key !== 'workBudgetYieldLimits' && key !== 'mapLatencyProfile' && key !== 'enforceFixtureProviderPacing', ); if (unknownKeys.length > 0) { return { error: `Unsupported testPolicyOverrides field "${unknownKeys[0]}".`, }; } const overrides: RuntimeTestPolicyOverrides = {}; if ('mapLatencyProfile' in record) { if (typeof record.mapLatencyProfile !== 'boolean') { return { error: 'testPolicyOverrides.mapLatencyProfile must be a boolean.', }; } overrides.mapLatencyProfile = record.mapLatencyProfile; } if ('enforceFixtureProviderPacing' in record) { if (typeof record.enforceFixtureProviderPacing !== 'boolean') { return { error: 'testPolicyOverrides.enforceFixtureProviderPacing must be a boolean.', }; } overrides.enforceFixtureProviderPacing = record.enforceFixtureProviderPacing; } for (const field of RUNTIME_TEST_POLICY_MS_FIELDS) { if (!(field in record)) continue; const parsed = readPositiveIntegerField({ value: record[field], path: `testPolicyOverrides.${field}`, max: MAX_RUNTIME_TEST_POLICY_MS, }); if (typeof parsed !== 'number') return parsed; (overrides as Record)[field] = parsed; } if ('workBudgetYieldLimits' in record) { const rawLimits = record.workBudgetYieldLimits; if ( rawLimits === null || typeof rawLimits !== 'object' || Array.isArray(rawLimits) ) { return { error: 'testPolicyOverrides.workBudgetYieldLimits must be an object.', }; } const limits: NonNullable< RuntimeTestPolicyOverrides['workBudgetYieldLimits'] > = {}; for (const [key, rawLimit] of Object.entries( rawLimits as Record, )) { if (!RUNTIME_TEST_POLICY_WORK_BUDGET_KEYS.has(key)) { return { error: `Unsupported testPolicyOverrides.workBudgetYieldLimits field "${key}".`, }; } const parsed = readPositiveIntegerField({ value: rawLimit, path: `testPolicyOverrides.workBudgetYieldLimits.${key}`, max: MAX_RUNTIME_TEST_POLICY_YIELD_LIMIT, }); if (typeof parsed !== 'number') return parsed; limits[key as keyof typeof limits] = parsed; } if (Object.keys(limits).length > 0) { overrides.workBudgetYieldLimits = limits; } } return Object.keys(overrides).length > 0 ? overrides : null; } export function validateRuntimeTestPolicyOverrides(input: { value: unknown; internalTokenHeader?: string | null; syntheticRunHeader?: string | null; }): ValidatedRuntimeTestPolicyOverrides { if (input.value === undefined || input.value === null) { return { ok: true, overrides: null }; } if (!runtimeTestSeamsAuthorized(input) || getRuntimeEnv() === 'prod') { return { ok: false, status: 403, error: 'Runtime test policy overrides are disabled. Set DEEPLINE_TEST_RUNTIME_SEAMS=1 only in dev/preview/CI harnesses.', }; } const parsed = parseRuntimeTestPolicyOverrides(input.value); if (parsed === null) { return { ok: true, overrides: null }; } if ('error' in parsed) { return { ok: false, status: 400, error: parsed.error }; } return { ok: true, overrides: parsed }; } export function readRuntimeTestFaultRegistry(input: { headerValue: string | null; internalTokenHeader?: string | null; syntheticRunHeader?: string | null; verifiedSyntheticExecutor?: boolean; }): ParsedRuntimeTestFaults { const headerValue = input.headerValue?.trim(); if (!headerValue) { return { ok: true, registry: null }; } if (!runtimeTestSeamsAuthorized(input) || getRuntimeEnv() === 'prod') { return { ok: false, status: 403, error: 'Runtime test seams are disabled. Set DEEPLINE_TEST_RUNTIME_SEAMS=1 only in dev/preview/CI harnesses.', }; } const parsed = parseRuntimeTestFaultHeader(headerValue); if ('error' in parsed) { return { ok: false, status: 400, error: parsed.error }; } return { ok: true, registry: { consume(name) { const count = parsed.get(name) ?? 0; if (count <= 0) return false; if (count === 1) { parsed.delete(name); } else { parsed.set(name, count - 1); } return true; }, remaining() { return Object.fromEntries(parsed.entries()); }, }, }; } export function validateRuntimeTestFaultHeader(input: { headerValue: string | null; internalTokenHeader?: string | null; syntheticRunHeader?: string | null; verifiedSyntheticExecutor?: boolean; }): ValidatedRuntimeTestFaultHeader { const parsed = parseRuntimeTestFaultCounts(input); if (parsed.ok === false) { return { ok: false, status: parsed.status, error: parsed.error }; } return { ok: true, headerValue: parsed.headerValue }; } export function parseRuntimeTestFaultCounts(input: { headerValue: string | null; internalTokenHeader?: string | null; syntheticRunHeader?: string | null; verifiedSyntheticExecutor?: boolean; }): ParsedRuntimeTestFaultCounts { const headerValue = input.headerValue?.trim(); if (!headerValue) { return { ok: true, counts: null, headerValue: null }; } if (!runtimeTestSeamsAuthorized(input) || getRuntimeEnv() === 'prod') { return { ok: false, status: 403, error: 'Runtime test seams are disabled. Set DEEPLINE_TEST_RUNTIME_SEAMS=1 only in dev/preview/CI harnesses.', }; } const parsed = parseRuntimeTestFaultHeader(headerValue); if ('error' in parsed) { return { ok: false, status: 400, error: parsed.error }; } return { ok: true, counts: Object.fromEntries(parsed.entries()), headerValue, }; }