import type { TelemetryConfig, TraceContextProvider } from "@github/copilot-sdk"; import { CopilotClient } from "@github/copilot-sdk"; import { Executor, ExecutorOptions, Stimulus, Trajectory } from "../index.js"; import { type CopilotSdkProviderConfig } from "../provider/copilot-provider.js"; import type { ExecutorTelemetryConfig, TelemetryConfigurableExecutor } from "./types.js"; export type { CopilotSdkProviderConfig }; /** * Run-scoped OpenTelemetry configuration for {@link CopilotSdkExecutor}. * * Exactly one delivery mode is selected per run: set {@link otlpEndpoint} to * export spans to an OTLP/HTTP endpoint, or {@link spansDir} to write spans to * per-incarnation JSON-lines files under that directory (the executor owns file * naming so concurrent/restarted client incarnations never share a file). * * @deprecated Prefer the shared {@link ExecutorTelemetryConfig}; this alias is * retained for backward compatibility and is structurally identical. */ export type CopilotSdkTelemetryConfig = ExecutorTelemetryConfig; export { toOtlpTraceUrl } from "./runtime-telemetry.js"; export { COPILOT_HOME_SETTINGS_JSON_ENV } from "./copilot-home.js"; /** * Executor-specific config for the `copilot-sdk` executor, authored under * `defaults.executor.config` in an eval spec. */ export interface CopilotSdkExecutorConfig { /** Bring-your-own-key (BYOK) model provider for the agent under test. */ provider?: CopilotSdkProviderConfig; } /** * Executor backed by the Copilot SDK. * * Concurrency: a single instance can run multiple `execute()` calls * concurrently. The underlying `CopilotClient` spawns one CLI server * process and multiplexes sessions over a single JSON-RPC connection. * Each `execute()` call creates an independent `CopilotSession` (keyed * by a unique UUID) with its own `Handler`, so events are routed * per-session with no cross-talk or shared mutable state. * * Recovery: if the client connection enters an error or disconnected * state, `execute()` will automatically force-stop and restart the * client so that subsequent calls use a fresh connection. Uses * exponential backoff with jitter (inspired by Orleans) and a circuit * breaker window (inspired by nginx's fail_timeout) to avoid hammering * a dead connection. */ export declare class CopilotSdkExecutor implements Executor, TelemetryConfigurableExecutor { private client; private shutdownPromise; private shutDownComplete; private shuttingDown; /** * Fresh, empty Copilot config home for this run, isolating the spawned * runtime from the developer's `~/.copilot` (see {@link createIsolatedCopilotHome} * and microsoft/vally#655). Resolved once, reused by every client incarnation, * and removed on {@link shutdown}. `undefined` means "not yet resolved" or * "use the host config" (opt-out). */ private isolatedCopilotHome; private isolatedCopilotHomeResolved; /** * True once {@link shutdown} has torn the executor down. While disposed, * {@link resolveIsolatedCopilotHome} must not create a new temp dir — shutdown * is the only thing that removes it and won't run again unless a new run * re-arms the executor — so it would otherwise leak an orphan directory. * {@link execute} clears this before resolving the home so a reused executor * still isolates. */ private disposed; private restartLock; private createClient; /** * Factory for short-lived per-run clients (used when `options.env` is set). */ private createEnvClient; private telemetryConfig; private frozen; /** Max ms shutdown will wait for the runtime's final span batch to flush. */ private readonly flushWindowMs; /** Wall-clock time the most recent turn ended, used to make the wait adaptive. */ private lastTurnEndedAt; private readonly traceContextStorage; private lastTraceContext; private activeTurns; private readonly traceContextProvider; private recentRestartTimestamps; private readonly perRunClients; private static readonly MAX_RESTARTS_IN_WINDOW; private static readonly RESTART_WINDOW_MS; private static readonly BACKOFF_SCHEDULE_MS; private static readonly BACKOFF_MAX_MS; private static readonly CLEANUP_TIMEOUT_MS; name: string; supportsPreparedWorkspace: boolean; supportsMultiTurn: boolean; supportsTurnCompletion: boolean; supportsSimulation: boolean; supportsAttachments: boolean; /** The underlying CopilotClient spawns a CLI server process, so `environment.env` is applied. */ readonly supportsEnvVars = true; private readonly reasoningEffortSupport; /** * Validate the `defaults.executor.config` block for this executor. Structural * checks only — secret resolution (`apiKeyEnv` / `bearerTokenEnv`) happens at * execution time. Throws with an actionable message on the first problem. */ validateConfig(config: unknown): void; constructor(options?: { createClient?: (telemetry: TelemetryConfig | undefined, traceContextProvider: TraceContextProvider) => CopilotClient; /** * Factory for short-lived per-run clients (used when `options.env` is set). * Tests inject a mock here to avoid spawning a real CLI process. */ createEnvClient?: (mergedEnv: NodeJS.ProcessEnv, telemetry: TelemetryConfig | undefined, traceContextProvider: TraceContextProvider) => CopilotClient; /** * Override the maximum time (ms) {@link shutdown} will wait for the runtime * to flush its final span batch when telemetry is active. The actual wait is * adaptive and never exceeds this. Tests pass `0` to skip waiting. */ flushWindowMs?: number; }); /** * Build the child-process environment for a spawned Copilot runtime, pinning * `COPILOT_HOME` to this run's isolated config home (unless the run opted out * via `EVALUATE_USE_HOST_COPILOT_HOME`). Isolation is unconditional — it does * not depend on session logging being enabled — so persisted local user * settings never leak into an eval. See microsoft/vally#655. */ private clientEnv; /** * Resolve (once) this run's isolated Copilot config home, or `undefined` when * the run opted out of isolation. Memoized so every client incarnation for * this executor shares the same empty home; removed in {@link shutdown}. */ private resolveIsolatedCopilotHome; /** * Configure run-scoped OpenTelemetry export for every client incarnation. * * Must be called before the first {@link execute}. The executor is a global * singleton resolved inside the eval×model loop, so this may be reached more * than once per run: a post-freeze call is a no-op when the config is * unchanged, and throws only on a genuinely conflicting reconfiguration. * * An empty config (neither `otlpEndpoint` nor `spansDir`) disables telemetry, * which avoids shutdown's final-flush wait for spans that will never be * produced. */ configureTelemetry(config: CopilotSdkTelemetryConfig): void; /** * Resolve the SDK {@link TelemetryConfig} for a fresh client incarnation. * * OTLP mode is endpoint-only; file mode generates a distinct `.jsonl` * path so concurrent clients and recovery-path restarts never share a file. */ private buildTelemetry; /** * Ensure the client is in a healthy state, restarting if needed. * Uses a lock so that concurrent callers don't all restart at once. */ private ensureHealthyClient; private admitEnvClient; private getClient; /** * Force-restart the client with exponential backoff and circuit breaker. * * Backoff: delays increase as [0, 1s, 2s, 4s, ...] with random jitter, * capped at 30s. Prevents thundering-herd restarts. * * Circuit breaker: if MAX_RESTARTS_IN_WINDOW restarts happen within * RESTART_WINDOW_MS, the circuit opens and restartClient() becomes a * no-op — the next execute() will fail naturally, letting the error * propagate to the caller rather than spin-looping. Restart timestamps * are aged out by time (not cleared on success) to enforce a true * sliding window. */ private restartClient; execute(stimulus: Stimulus, options: ExecutorOptions): Promise; /** * Per-run teardown for `execute()`: stops a per-run client with a bounded * timeout (falling back to force-stop), or — if `shutdown()` claimed the * client first — transfers ownership of the settings home to that path * instead of deleting a directory it may still be reading from. Always * removes an unclaimed settings home, even if the client was never admitted * (e.g. admission itself threw). */ private finalizePerRunResources; /** * Fails fast when a concrete `reasoningEffort` is requested for a model that * definitively does not support it, so the run stops with a clear, actionable * error instead of the runtime's lower-level `session.create` rejection (the * Copilot runtime rejects an unsupported reasoning effort rather than silently * ignoring it). Only an explicit negative throws — the model is listed by * `listModels()` and advertises `reasoningEffort: false`. Unknown models, * missing metadata, an unset model (host default, whose identity we can't * attribute), and any lookup failure stay silent and let the runtime be the * final arbiter — a capability probe must never fail a run it can't * definitively rule out. Support is cached per model so `listModels()` is * probed at most once per model, regardless of outcome — the in-flight probe * promise is cached, so concurrent trials for the same model share one lookup. */ private assertReasoningEffortSupported; /** * Resolves whether `model` advertises reasoning-effort support via a single * `listModels()` lookup. Returns `undefined` (not a verdict) when the lookup * fails, so the caller can retry later instead of caching a transient error. */ private resolveReasoningEffortSupport; private runTurn; private withTraceContext; shutdown(): Promise; /** * Compute how long {@link shutdown} should wait for the runtime's final span * batch to flush: the flush window minus the time already elapsed since the * last turn ended, clamped to `[0, flushWindowMs]`. Returns `0` when telemetry * is inactive, no turn has run, or the window has already elapsed. */ private remainingFlushWaitMs; } /** * Validate a {@link CopilotSdkExecutorConfig} (the `defaults.executor.config` * block for the copilot-sdk executor). Structural checks only — secret * resolution happens at execution time. Throws on the first problem found. */ export declare function validateCopilotSdkExecutorConfig(value: unknown, label?: string): void; //# sourceMappingURL=copilot-sdk-executor.d.ts.map