/** * Runtime availability policy shared by the scheduler, Fly workers, receipt * gateway, and sandbox runner. These are platform controls, not Play options: * a Play author must never be able to make an unsafe side effect retry longer * or more often. * * Defaults are deliberately compiled into the runtime so a missing deploy * variable is safe. A changed value is accepted only when an incident operator * also sets `DEEPLINE_RUNTIME_RELIABILITY_SEV_OVERRIDE=enabled`. This makes an * emergency tuning change explicit, auditable in deployment configuration, and * impossible to smuggle through an ordinary per-run request. */ export const RUNTIME_RELIABILITY_SEV_OVERRIDE_ENV = 'DEEPLINE_RUNTIME_RELIABILITY_SEV_OVERRIDE'; export const RUNTIME_RELIABILITY_ENV = { engineMaxAttempts: 'DEEPLINE_RUNTIME_ENGINE_MAX_ATTEMPTS', engineRetryBaseSeconds: 'DEEPLINE_RUNTIME_ENGINE_RETRY_BASE_SECONDS', engineRetryMaxSeconds: 'DEEPLINE_RUNTIME_ENGINE_RETRY_MAX_SECONDS', schedulerSubmitCapacityMaxAttempts: 'DEEPLINE_RUNTIME_SCHEDULER_SUBMIT_CAPACITY_MAX_ATTEMPTS', claimLeaseSeconds: 'DEEPLINE_RUNTIME_CLAIM_LEASE_SECONDS', schedulerDbAcquireTimeoutMs: 'DEEPLINE_RUNTIME_SCHEDULER_DB_ACQUIRE_TIMEOUT_MS', schedulerDbQueryTimeoutMs: 'DEEPLINE_RUNTIME_SCHEDULER_DB_QUERY_TIMEOUT_MS', schedulerDbRetryAttempts: 'DEEPLINE_RUNTIME_SCHEDULER_DB_RETRY_ATTEMPTS', schedulerDbRetryBaseDelayMs: 'DEEPLINE_RUNTIME_SCHEDULER_DB_RETRY_BASE_DELAY_MS', schedulerDbRetryMaxDelayMs: 'DEEPLINE_RUNTIME_SCHEDULER_DB_RETRY_MAX_DELAY_MS', schedulerDbCircuitOpenMs: 'DEEPLINE_RUNTIME_SCHEDULER_DB_CIRCUIT_OPEN_MS', recoveryMaxDefers: 'DEEPLINE_RUNTIME_RECOVERY_MAX_DEFERS', recoveryHorizonMs: 'DEEPLINE_RUNTIME_RECOVERY_HORIZON_MS', recoveryDeferBaseSeconds: 'DEEPLINE_RUNTIME_RECOVERY_DEFER_BASE_SECONDS', recoveryDeferMaxSeconds: 'DEEPLINE_RUNTIME_RECOVERY_DEFER_MAX_SECONDS', recoveryCircuitOpenMs: 'DEEPLINE_RUNTIME_RECOVERY_CIRCUIT_OPEN_MS', recoveryCircuitProbeLeaseMs: 'DEEPLINE_RUNTIME_RECOVERY_CIRCUIT_PROBE_LEASE_MS', sandboxRunnerReadyTimeoutMs: 'DEEPLINE_RUNTIME_SANDBOX_RUNNER_READY_TIMEOUT_MS', modalSandboxCreateTimeoutMs: 'DEEPLINE_RUNTIME_MODAL_SANDBOX_CREATE_TIMEOUT_MS', daytonaCommandRecoveryTimeoutMs: 'DEEPLINE_RUNTIME_DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS', daytonaCommandRecoveryPollMs: 'DEEPLINE_RUNTIME_DAYTONA_COMMAND_RECOVERY_POLL_MS', runnerControlRequestTimeoutMs: 'DEEPLINE_RUNTIME_RUNNER_CONTROL_REQUEST_TIMEOUT_MS', runnerControlMaxAttempts: 'DEEPLINE_RUNTIME_RUNNER_CONTROL_MAX_ATTEMPTS', terminalPushRequestTimeoutMs: 'DEEPLINE_RUNTIME_TERMINAL_PUSH_REQUEST_TIMEOUT_MS', terminalPushMaxAttempts: 'DEEPLINE_RUNTIME_TERMINAL_PUSH_MAX_ATTEMPTS', egressFetchMaxAttempts: 'DEEPLINE_RUNTIME_EGRESS_FETCH_MAX_ATTEMPTS', egressFetchHeadersTimeoutMs: 'DEEPLINE_RUNTIME_EGRESS_FETCH_HEADERS_TIMEOUT_MS', egressFetchBodyTimeoutMs: 'DEEPLINE_RUNTIME_EGRESS_FETCH_BODY_TIMEOUT_MS', egressFetchTotalTimeoutMs: 'DEEPLINE_RUNTIME_EGRESS_FETCH_TOTAL_TIMEOUT_MS', } as const; export type RuntimeReliabilityPolicy = Readonly<{ sevOverrideEnabled: boolean; engine: Readonly<{ maxAttempts: number; retryBaseSeconds: number; retryMaxSeconds: number; }>; schedulerSubmit: Readonly<{ capacityMaxAttempts: number }>; worker: Readonly<{ claimLeaseSeconds: number }>; schedulerDatabase: Readonly<{ acquireTimeoutMs: number; queryTimeoutMs: number; retryAttempts: number; retryBaseDelayMs: number; retryMaxDelayMs: number; circuitOpenMs: number; }>; recovery: Readonly<{ maxDefers: number; horizonMs: number; deferBaseSeconds: number; deferMaxSeconds: number; circuitOpenMs: number; circuitProbeLeaseMs: number; }>; sandbox: Readonly<{ modalSandboxCreateTimeoutMs: number; runnerReadyTimeoutMs: number; daytonaCommandRecoveryTimeoutMs: number; daytonaCommandRecoveryPollMs: number; }>; gateway: Readonly<{ runnerControlRequestTimeoutMs: number; runnerControlMaxAttempts: number; terminalPushRequestTimeoutMs: number; terminalPushMaxAttempts: number; }>; egress: Readonly<{ fetchMaxAttempts: number; fetchHeadersTimeoutMs: number; fetchBodyTimeoutMs: number; fetchTotalTimeoutMs: number; }>; }>; type Env = Readonly>; type NumberSetting = Readonly<{ name: string; defaultValue: number; min: number; max: number; }>; function readProcessEnvironment(): Env { return typeof process === 'undefined' ? {} : (process.env ?? {}); } function enabled(raw: string | undefined): boolean { return raw?.trim().toLowerCase() === 'enabled'; } function resolveSetting( env: Env, sevOverrideEnabled: boolean, setting: NumberSetting, ): number { const raw = env[setting.name]?.trim(); if (!raw) return setting.defaultValue; if (!/^\d+$/.test(raw)) { throw new Error( `${setting.name} must be an integer between ${setting.min} and ${setting.max}.`, ); } const value = Number(raw); if ( !Number.isSafeInteger(value) || value < setting.min || value > setting.max ) { throw new Error( `${setting.name} must be an integer between ${setting.min} and ${setting.max}.`, ); } if (value !== setting.defaultValue && !sevOverrideEnabled) { throw new Error( `${setting.name} differs from its safe platform default. Set ${RUNTIME_RELIABILITY_SEV_OVERRIDE_ENV}=enabled only for an active SEV.`, ); } return value; } function setting( name: string, defaultValue: number, min: number, max: number, ): NumberSetting { return { name, defaultValue, min, max }; } /** Pure resolver so tests can prove the deployment guard without mutating env. */ export function resolveRuntimeReliabilityPolicy( env: Env = readProcessEnvironment(), ): RuntimeReliabilityPolicy { const sevOverrideEnabled = enabled(env[RUNTIME_RELIABILITY_SEV_OVERRIDE_ENV]); const read = (name: string, defaultValue: number, min: number, max: number) => resolveSetting( env, sevOverrideEnabled, setting(name, defaultValue, min, max), ); const policy: RuntimeReliabilityPolicy = { sevOverrideEnabled, engine: { maxAttempts: read(RUNTIME_RELIABILITY_ENV.engineMaxAttempts, 5, 1, 8), retryBaseSeconds: read( RUNTIME_RELIABILITY_ENV.engineRetryBaseSeconds, 5, 1, 60, ), retryMaxSeconds: read( RUNTIME_RELIABILITY_ENV.engineRetryMaxSeconds, 60, 5, 300, ), }, schedulerSubmit: { capacityMaxAttempts: read( RUNTIME_RELIABILITY_ENV.schedulerSubmitCapacityMaxAttempts, 8, 1, 12, ), }, worker: { claimLeaseSeconds: read( RUNTIME_RELIABILITY_ENV.claimLeaseSeconds, 120, 30, 600, ), }, schedulerDatabase: { acquireTimeoutMs: read( RUNTIME_RELIABILITY_ENV.schedulerDbAcquireTimeoutMs, 5_000, 1_000, 30_000, ), queryTimeoutMs: read( RUNTIME_RELIABILITY_ENV.schedulerDbQueryTimeoutMs, 30_000, 5_000, 120_000, ), retryAttempts: read( RUNTIME_RELIABILITY_ENV.schedulerDbRetryAttempts, 3, 1, 6, ), retryBaseDelayMs: read( RUNTIME_RELIABILITY_ENV.schedulerDbRetryBaseDelayMs, 50, 10, 5_000, ), retryMaxDelayMs: read( RUNTIME_RELIABILITY_ENV.schedulerDbRetryMaxDelayMs, 1_000, 50, 30_000, ), circuitOpenMs: read( RUNTIME_RELIABILITY_ENV.schedulerDbCircuitOpenMs, 2_000, 500, 60_000, ), }, recovery: { maxDefers: read(RUNTIME_RELIABILITY_ENV.recoveryMaxDefers, 8, 1, 12), horizonMs: read( RUNTIME_RELIABILITY_ENV.recoveryHorizonMs, 30 * 60_000, 60_000, 2 * 60 * 60_000, ), deferBaseSeconds: read( RUNTIME_RELIABILITY_ENV.recoveryDeferBaseSeconds, 15, 1, 300, ), deferMaxSeconds: read( RUNTIME_RELIABILITY_ENV.recoveryDeferMaxSeconds, 300, 15, 900, ), circuitOpenMs: read( RUNTIME_RELIABILITY_ENV.recoveryCircuitOpenMs, 60_000, 5_000, 5 * 60_000, ), circuitProbeLeaseMs: read( RUNTIME_RELIABILITY_ENV.recoveryCircuitProbeLeaseMs, 60_000, 5_000, 5 * 60_000, ), }, sandbox: { modalSandboxCreateTimeoutMs: read( RUNTIME_RELIABILITY_ENV.modalSandboxCreateTimeoutMs, 60_000, 5_000, 5 * 60_000, ), runnerReadyTimeoutMs: read( RUNTIME_RELIABILITY_ENV.sandboxRunnerReadyTimeoutMs, 30_000, 5_000, 5 * 60_000, ), daytonaCommandRecoveryTimeoutMs: read( RUNTIME_RELIABILITY_ENV.daytonaCommandRecoveryTimeoutMs, 60_000, 5_000, 5 * 60_000, ), daytonaCommandRecoveryPollMs: read( RUNTIME_RELIABILITY_ENV.daytonaCommandRecoveryPollMs, 2_000, 100, 30_000, ), }, gateway: { runnerControlRequestTimeoutMs: read( RUNTIME_RELIABILITY_ENV.runnerControlRequestTimeoutMs, 20_000, 1_000, 120_000, ), runnerControlMaxAttempts: read( RUNTIME_RELIABILITY_ENV.runnerControlMaxAttempts, 3, 1, 6, ), terminalPushRequestTimeoutMs: read( RUNTIME_RELIABILITY_ENV.terminalPushRequestTimeoutMs, 15_000, 1_000, 120_000, ), terminalPushMaxAttempts: read( RUNTIME_RELIABILITY_ENV.terminalPushMaxAttempts, 5, 1, 8, ), }, egress: { fetchMaxAttempts: read( RUNTIME_RELIABILITY_ENV.egressFetchMaxAttempts, 3, 1, 5, ), fetchHeadersTimeoutMs: read( RUNTIME_RELIABILITY_ENV.egressFetchHeadersTimeoutMs, 30_000, 1_000, 5 * 60_000, ), fetchBodyTimeoutMs: read( RUNTIME_RELIABILITY_ENV.egressFetchBodyTimeoutMs, 30_000, 1_000, 5 * 60_000, ), fetchTotalTimeoutMs: read( RUNTIME_RELIABILITY_ENV.egressFetchTotalTimeoutMs, 120_000, 5_000, 15 * 60_000, ), }, }; if (policy.engine.retryMaxSeconds < policy.engine.retryBaseSeconds) { throw new Error('Engine retry maximum must be at least the retry base.'); } if ( policy.schedulerDatabase.retryMaxDelayMs < policy.schedulerDatabase.retryBaseDelayMs ) { throw new Error( 'Scheduler DB retry maximum must be at least the retry base.', ); } if (policy.recovery.deferMaxSeconds < policy.recovery.deferBaseSeconds) { throw new Error('Recovery defer maximum must be at least the defer base.'); } if ( policy.recovery.circuitProbeLeaseMs < policy.sandbox.modalSandboxCreateTimeoutMs ) { throw new Error( 'Recovery circuit probe lease must cover the Modal sandbox create deadline.', ); } if ( policy.egress.fetchTotalTimeoutMs < policy.egress.fetchHeadersTimeoutMs || policy.egress.fetchTotalTimeoutMs < policy.egress.fetchBodyTimeoutMs ) { throw new Error( 'Egress total timeout must cover both headers and body deadlines.', ); } return policy; } export const RUNTIME_RELIABILITY_POLICY = resolveRuntimeReliabilityPolicy();