/**
 * Aspect runner — executes a module's base-module aspect across
 * the fleet.
 *
 * Per openspec/specs/base-module-aspects/spec.md Phase 1: when a module with an approved
 * `base_module_aspect` triggers a fan-out event, the runner walks
 * every non-`api_only` system in the aspect's `applicable_zones`,
 * materializes an Ansible inventory + playbook in a scratch
 * directory, copies the aspect role files there, and invokes
 * `executeAnsible` against the result. The existing Ansible
 * machinery does the actual SSH + playbook execution; the runner
 * only sets up the per-aspect Ansible workspace.
 *
 * Fan-out covers BOTH machine-pool systems and container_service
 * LXCs in the aspect's zones (ISS-0028): machines come from
 * `getSystemsByZone`, LXCs from `getContainerSystemsInZones`. LXCs
 * authenticate via the ambient operator key (no per-host key).
 *
 * Scope NOT covered here:
 *  - Proxmox `nameserver` reconciliation is SC5.
 *  - Trigger wiring (when `on_install` / `on_new_system_in_zone`
 *    etc. fire) is SC4. SC3 ships the pure execution surface so
 *    SC4 can call it from the deploy planner.
 *  - Capability data injection into aspect host_vars is Phase 2+.
 */

import { existsSync } from 'node:fs';
import { cp, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { eq } from 'drizzle-orm';
import { type InventoryHost, generateHostsIni } from '../ansible/inventory';
import { log } from '../cli/prompts';
import { getDb } from '../db/client';
import { type NetworkZone, modules } from '../db/schema';
import type { BaseModuleAspect, BaseModuleAspectTrigger, ModuleManifest } from '../manifest/schema';
import type { Machine } from '../types/infrastructure';
import {
  checkAspectApproval,
  computeAspectScopeHash,
  recordAspectConsent,
} from './aspect-approvals';
import { resolveAspectTemplateRecord } from './aspect-template-resolver';
import {
  type AspectConsentReply,
  type AspectRequiredPayload,
  EVENT_TYPES,
  busInterviewGuarded,
} from './bus-interview';
import { type AnsibleHostRecap, executeAnsible, parseAnsibleRecap } from './deploy-ansible';
import { getContainerSystemsInZones } from './deployed-systems';
import { getSystemsByZone } from './machine-pool';
import { executeProxmoxReconcile, planProxmoxReconcile } from './proxmox-reconcile';
import { LOCAL_MACHINE_IP, writeTemporarySshKey } from './ssh-key-manager';

type DbClient = ReturnType<typeof getDb>;

/**
 * A single host an aspect fans out to — a machine-pool box OR a
 * container_service LXC (ISS-0028). The two differ only in SSH auth:
 *
 *   - machine: `machineId` is set; its per-machine key is materialized via
 *     `writeTemporarySshKey(machineId)` and pinned in the inventory row.
 *   - LXC: `machineId` is undefined; it authenticates via the ambient operator
 *     key (the one terraform injected at provision time), exactly as a normal
 *     LXC module deploy does — so the inventory row carries no key file.
 */
export interface AspectTarget {
  hostname: string;
  ipAddress: string;
  sshUser: string;
  zone: NetworkZone;
  /** Set for machine-pool targets only. Drives per-machine SSH key staging. */
  machineId?: string;
}

export interface AspectFanOutPlan {
  /** Systems the aspect will run on (machines + container_service LXCs). */
  targetSystems: AspectTarget[];
  /** Machines that matched the zones but were excluded (api_only, etc.). */
  skipped: Array<{ machine: Machine; reason: string }>;
}

export interface AspectRunResult {
  success: boolean;
  /** Ansible recap-level summary, or empty string if nothing ran. */
  output: string;
  /** Populated when success === false. */
  error?: string;
  /** The plan that was executed, for caller logging / event emission. */
  plan: AspectFanOutPlan;
  /**
   * Per-host `PLAY RECAP` counters. Empty when Ansible produced no recap — a
   * state callers must NOT read as success. `verifyAspectCoverage` classifies
   * from this.
   */
  recap: AnsibleHostRecap[];
}

export interface AspectRunOptions {
  /** Which trigger caused this fan-out. Recorded for logging. */
  trigger: BaseModuleAspectTrigger;
  /**
   * Hostnames to exclude from fan-out, on top of `api_only`. The
   * primary deploy's own host(s) typically don't need the aspect
   * (per openspec/specs/base-module-aspects/spec.md D3 — aspect authors handle this when
   * they want it, the framework doesn't force a skip).
   */
  excludeHostnames?: string[];
  /**
   * Restrict the fan-out to exactly these hostnames, intersected with the
   * zone-derived target set. Used by the inbound reconcile
   * (`reconcileAspectsForSystems`), which applies an aspect to the systems a
   * deploy just created rather than to the whole fleet. Narrows only — a
   * hostname the aspect's zones do not cover is still not a target.
   */
  onlyHostnames?: string[];
  /**
   * Run Ansible in CHECK mode: evaluate the role against each target without
   * changing anything. Used by `verifyAspectCoverage` to ask the HOST whether
   * an aspect is applied, rather than consulting a stored claim that it once
   * ran (celilo#902 design D6).
   */
  check?: boolean;
  /**
   * Override for noInteractive mode passed to executeAnsible.
   * Defaults to `true` because aspect fan-outs run as part of
   * larger orchestration (deploy flow) — there's no operator
   * waiting at this specific step.
   */
  noInteractive?: boolean;
}

/**
 * Planning-phase function (Rule 10.1): resolves which systems the
 * aspect will target, without executing anything. Pure aside from
 * the database read.
 */
export async function planAspectFanOut(
  aspect: BaseModuleAspect,
  options: Pick<AspectRunOptions, 'excludeHostnames' | 'onlyHostnames'> = {},
): Promise<AspectFanOutPlan> {
  // `onlyHostnames` narrows, never widens: it is intersected with the
  // zone-derived set below rather than replacing it, so an inbound reconcile
  // cannot apply an aspect to a system outside its `applicable_zones`.
  const only = options.onlyHostnames ? new Set(options.onlyHostnames) : undefined;
  // Machine-pool systems in the zones (api_only recorded as skips for
  // observability). excludeHostnames is applied by getSystemsByZone.
  const allInZones = await getSystemsByZone(aspect.applicable_zones, {
    excludeApiOnly: false,
    excludeHostnames: options.excludeHostnames,
  });

  const targetSystems: AspectTarget[] = [];
  const skipped: AspectFanOutPlan['skipped'] = [];
  const seen = new Set<string>();
  for (const m of allInZones) {
    if (m.apiOnly) {
      skipped.push({ machine: m, reason: 'api_only' });
      continue;
    }
    if (only && !only.has(m.hostname)) continue;
    targetSystems.push({
      hostname: m.hostname,
      ipAddress: m.ipAddress,
      sshUser: m.sshUser,
      zone: m.zone,
      machineId: m.id,
    });
    seen.add(m.hostname);
  }

  // Container_service LXCs in the same zones (ISS-0028). The machines-only
  // fan-out skipped these; without them an aspect like dns-client-config can't
  // reach caddy/celilo-registry. They auth via the ambient operator key, so no
  // machineId / per-host key.
  const exclude = new Set(options.excludeHostnames ?? []);
  for (const sys of getContainerSystemsInZones(aspect.applicable_zones, getDb())) {
    if (exclude.has(sys.hostname) || seen.has(sys.hostname)) continue;
    if (only && !only.has(sys.hostname)) continue;
    targetSystems.push({
      hostname: sys.hostname,
      ipAddress: sys.ipv4_address,
      sshUser: 'root',
      zone: sys.zone as NetworkZone,
    });
    seen.add(sys.hostname);
  }

  return { targetSystems, skipped };
}

/**
 * Materialize the per-aspect Ansible workspace in a temp dir:
 *
 *   <tmp>/
 *     ansible/
 *       inventory/
 *         hosts.ini
 *         host_vars/<hostname>.yml      (target_zone fact)
 *         group_vars/all/aspect_vars.yml  (resolved ansible_vars
 *                                          from the manifest, if any)
 *       playbook.yml                    (synthesized)
 *       roles/<aspect_role>/            (copied from the module's
 *                                        base-module-aspect/ tree)
 *
 * Returns the temp dir path; caller is responsible for cleanup.
 * Throws if the module's aspect role doesn't exist on disk.
 *
 * When `aspect.ansible_vars` is set, each template resolves against
 * the providing module's context (its module_configs, capability
 * data, system_config) and lands in group_vars/all/aspect_vars.yml
 * — readable from the role as `{{ var_name }}`.
 */
export async function materializeAspectAnsible(args: {
  aspect: BaseModuleAspect;
  /** Absolute path to the module's imported source dir. The aspect
   *  role is expected at `<moduleSourcePath>/base-module-aspect/ansible/roles/<aspect.ansible_role>`. */
  moduleSourcePath: string;
  targetSystems: AspectTarget[];
  /** Provider module ID — used to resolve $self / $capability /
   *  $system references in ansible_vars. */
  providerModuleId?: string;
  /** DB client for context resolution. Required when ansible_vars
   *  is declared on the aspect. */
  db?: DbClient;
}): Promise<string> {
  const { aspect, moduleSourcePath, targetSystems, providerModuleId, db } = args;
  const roleSrcDir = join(
    moduleSourcePath,
    'base-module-aspect',
    'ansible',
    'roles',
    aspect.ansible_role,
  );
  if (!existsSync(roleSrcDir)) {
    throw new Error(
      `Aspect role not found at ${roleSrcDir}. The module declared base_module_aspect.ansible_role='${aspect.ansible_role}' but the corresponding directory is missing.`,
    );
  }

  const workDir = await mkdtemp(join(tmpdir(), 'celilo-aspect-'));
  const ansibleDir = join(workDir, 'ansible');
  const inventoryDir = join(ansibleDir, 'inventory');
  const hostVarsDir = join(inventoryDir, 'host_vars');
  const rolesDir = join(ansibleDir, 'roles');
  await mkdir(hostVarsDir, { recursive: true });
  await mkdir(rolesDir, { recursive: true });

  // Stage each target's SSH access and build inventory rows. Machines pin
  // their per-machine key; LXCs (machineId undefined) leave it off and use the
  // ambient operator key — same as a normal LXC deploy (ISS-0028).
  const inventoryHosts: InventoryHost[] = [];
  for (const t of targetSystems) {
    // The management box registers itself in the machine pool as 127.0.0.1 and
    // stores NO ssh key -- it does not need one, because Ansible reaches it with
    // the local connection. Mirrors the module-deploy inventory path
    // (ansible/inventory.ts). Without this the fan-out tried `ssh root@127.0.0.1`
    // with an empty key file and the host failed UNREACHABLE with an opaque
    // "error in libcrypto", so celilo could not configure its own resolv.conf.
    const isLocal = t.ipAddress === LOCAL_MACHINE_IP;
    const keyPath = t.machineId && !isLocal ? await writeTemporarySshKey(t.machineId) : undefined;
    inventoryHosts.push({
      hostname: t.hostname,
      ansibleHost: t.ipAddress,
      ansibleUser: t.sshUser,
      groups: ['aspect_targets', t.zone],
      local: isLocal,
      ansibleSshPrivateKeyFile: keyPath,
    });

    // Per-host vars: target_zone is the only fact the framework
    // guarantees today (per D3). Capability-data / module-config
    // injection is Phase 2+.
    const hostVars = `---\n# Aspect host vars for ${t.hostname}\ntarget_zone: ${t.zone}\n`;
    await writeFile(join(hostVarsDir, `${t.hostname}.yml`), hostVars, 'utf-8');
  }

  // hosts.ini groups every target under 'aspect_targets' and the
  // per-system zone. The playbook (below) targets 'aspect_targets'.
  const hostsIni = generateHostsIni(inventoryHosts);
  await writeFile(join(inventoryDir, 'hosts.ini'), hostsIni, 'utf-8');

  // Resolve and write aspect ansible_vars (if any) to
  // group_vars/all/aspect_vars.yml. Every target reads these as
  // `{{ var_name }}` from the role. Values resolve against the
  // providing module's context — its module_configs, capability
  // data, system_config — so the role sees concrete strings.
  if (aspect.ansible_vars && Object.keys(aspect.ansible_vars).length > 0) {
    if (!providerModuleId || !db) {
      throw new Error(
        'materializeAspectAnsible: aspect declares ansible_vars but providerModuleId/db were not supplied. This is a framework bug.',
      );
    }
    const resolved = await resolveAspectTemplateRecord(
      aspect.ansible_vars,
      providerModuleId,
      db,
      'base_module_aspect.ansible_vars',
    );
    const groupVarsAllDir = join(inventoryDir, 'group_vars', 'all');
    await mkdir(groupVarsAllDir, { recursive: true });
    const lines = [
      '---',
      `# Resolved base_module_aspect.ansible_vars for ${providerModuleId}`,
      ...Object.entries(resolved).map(([k, v]) => `${k}: ${JSON.stringify(v)}`),
      '',
    ];
    await writeFile(join(groupVarsAllDir, 'aspect_vars.yml'), lines.join('\n'), 'utf-8');
  }

  // Copy the role tree into the staging dir so Ansible can find it
  // via the default role path. cp -r equivalent; the role directory
  // structure (tasks/, templates/, handlers/, vars/, defaults/)
  // comes along intact.
  await cp(roleSrcDir, join(rolesDir, aspect.ansible_role), { recursive: true });

  // Synthesize the playbook. One play, targeting every host in the
  // 'aspect_targets' group, invoking the single role. Become true
  // because aspect roles typically modify /etc/* files.
  const playbook = [
    '---',
    `- name: Aspect '${aspect.ansible_role}' fan-out`,
    '  hosts: aspect_targets',
    '  become: true',
    '  gather_facts: true',
    '  roles:',
    `    - role: ${aspect.ansible_role}`,
    '',
  ].join('\n');
  await writeFile(join(ansibleDir, 'playbook.yml'), playbook, 'utf-8');

  return workDir;
}

/**
 * Execution-phase function: orchestrates plan + materialize + run.
 *
 * Failure semantics (openspec/specs/base-module-aspects/spec.md D4): aspects are idempotent
 * and forward-progress only. A failed fan-out is reported and the
 * partial state (some systems updated, others not) is preserved —
 * no rollback. The caller (deploy planner, in SC4) decides how to
 * surface the failure.
 */
export async function runAspectFanOut(args: {
  moduleId: string;
  aspect: BaseModuleAspect;
  moduleSourcePath: string;
  options: AspectRunOptions;
  db: DbClient;
}): Promise<AspectRunResult> {
  const { moduleId, aspect, moduleSourcePath, options, db } = args;

  const plan = await planAspectFanOut(aspect, {
    excludeHostnames: options.excludeHostnames,
    onlyHostnames: options.onlyHostnames,
  });

  if (plan.targetSystems.length === 0) {
    log.info(
      `Aspect fan-out for '${moduleId}' (${options.trigger}): no eligible systems in zones [${aspect.applicable_zones.join(', ')}]`,
    );
    return { success: true, output: '', plan, recap: [] };
  }

  log.info(
    `Aspect fan-out for '${moduleId}' (${options.trigger}): ${plan.targetSystems.length} target(s) in zones [${aspect.applicable_zones.join(', ')}]`,
  );

  let workDir: string | undefined;
  try {
    workDir = await materializeAspectAnsible({
      aspect,
      moduleSourcePath,
      targetSystems: plan.targetSystems,
      providerModuleId: moduleId,
      db,
    });

    const result = await executeAnsible(workDir, {
      noInteractive: options.noInteractive ?? true,
      check: options.check,
    });

    // Proxmox reconciliation (D5): if the aspect declares
    // proxmox_reconcile.tfvars and the fan-out plan includes
    // Proxmox-provisioned LXCs, surface what the persisted
    // terraform config WOULD need to look like. Currently
    // observation-only (see proxmox-reconcile.ts header); when the
    // persistence layer lands the planning here stays unchanged.
    //
    // Only attempt reconciliation if the Ansible run succeeded —
    // there's no point warning about persisted-config drift if
    // the running config didn't update.
    if (result.success && aspect.proxmox_reconcile) {
      try {
        const reconcilePlan = await planProxmoxReconcile({
          aspect,
          providerModuleId: moduleId,
          db,
        });
        executeProxmoxReconcile(reconcilePlan);
      } catch (err) {
        // Reconciliation planning failed (e.g., capability data
        // missing). Don't fail the fan-out — the running config
        // is already updated. Just warn.
        log.warn(
          `Proxmox reconciliation planning failed for '${moduleId}': ${err instanceof Error ? err.message : String(err)}`,
        );
      }
    }

    return {
      success: result.success,
      output: result.output,
      error: result.error,
      plan,
      recap: parseAnsibleRecap(result.output),
    };
  } finally {
    if (workDir) {
      try {
        await rm(workDir, { recursive: true, force: true });
      } catch {
        // Best-effort cleanup. If rm fails the tmpdir GC will handle it.
      }
    }
  }
}

/**
 * Reasons the deploy planner might skip a fan-out without raising
 * an error. Each is a "this is fine, just not applicable" outcome
 * that the planner should log but not treat as a deploy failure.
 */
export type AspectSkipReason =
  | 'no_aspect' // module didn't declare base_module_aspect
  | 'trigger_not_declared' // aspect.triggers doesn't include this trigger
  | 'no_approval' // operator hasn't approved (D2)
  | 'denied' // operator explicitly refused consent (ISS-0027)
  | 'scope_changed'; // approval exists but applicable_zones/triggers diverged (D7)

/**
 * Asks the operator to approve (true) or refuse (false) a module's
 * base-module aspect. Injectable so unit tests don't drive the bus;
 * the default (`requestAspectConsentViaBus`) emits an
 * `aspect.required.<module>.<role>` interview question and waits for
 * a responder (terminal, `celilo events reply`, the celilo-deploy
 * skill). See ISS-0027.
 */
export type AspectConsentRequest = (args: {
  moduleId: string;
  version: string;
  aspect: BaseModuleAspect;
  trigger: BaseModuleAspectTrigger;
  reason: 'no_approval' | 'scope_changed';
}) => Promise<boolean>;

async function requestAspectConsentViaBus(args: {
  moduleId: string;
  version: string;
  aspect: BaseModuleAspect;
  trigger: BaseModuleAspectTrigger;
  reason: 'no_approval' | 'scope_changed';
}): Promise<boolean> {
  const payload: AspectRequiredPayload = {
    module: args.moduleId,
    role: args.aspect.ansible_role,
    zones: args.aspect.applicable_zones,
    triggers: args.aspect.triggers,
    trigger: args.trigger,
    reason: args.reason,
  };
  const reply = await busInterviewGuarded<AspectConsentReply>(
    EVENT_TYPES.aspectRequired(args.moduleId, args.aspect.ansible_role),
    payload,
  );
  return reply.consented === true;
}

export interface AspectGlueResult {
  ran: boolean;
  /** Populated when `ran === true`. */
  success?: boolean;
  /** Populated when `ran === false`. */
  reason?: AspectSkipReason;
  /** Populated when `ran === true` — the fan-out plan + Ansible recap. */
  runResult?: AspectRunResult;
}

/**
 * The consent gate, shared by both fan-out directions.
 *
 * Already-approved → run. Already-DENIED → skip silently: the operator made a
 * durable decision and re-prompting every deploy would be nagging (ISS-0027).
 * Only the undecided ('no_approval') and stale ('scope_changed') states warrant
 * an interview, and the decision is persisted either way so a denial isn't
 * re-asked next deploy.
 *
 * Extracted so `reconcileAspectsForSystems` cannot drift from
 * `maybeRunAspectForTrigger`: an inbound reconcile that skipped this would
 * apply an aspect the operator never approved, or one they refused.
 */
async function ensureAspectConsent(args: {
  moduleId: string;
  version: string;
  aspect: BaseModuleAspect;
  trigger: BaseModuleAspectTrigger;
  db: DbClient;
  requestConsent?: AspectConsentRequest;
}): Promise<{ consented: true } | { consented: false; reason: AspectSkipReason }> {
  const { moduleId, version, aspect, trigger, db } = args;
  const approvalStatus = checkAspectApproval(moduleId, version, aspect, db);

  if (approvalStatus === 'denied') {
    return { consented: false, reason: 'denied' };
  }
  if (approvalStatus === 'no_approval' || approvalStatus === 'scope_changed') {
    // ISS-0027: don't silently skip a declared aspect. Interview the operator
    // for consent on the bus and WAIT. A responder (terminal, `events reply`,
    // the celilo-deploy skill) approves or denies.
    const requestConsent = args.requestConsent ?? requestAspectConsentViaBus;
    const consented = await requestConsent({
      moduleId,
      version,
      aspect,
      trigger,
      reason: approvalStatus,
    });
    recordAspectConsent({
      moduleId,
      version,
      scopeHash: computeAspectScopeHash(aspect),
      approver: process.env.USER ?? null,
      consented,
      db,
    });
    if (!consented) {
      log.warn(`Aspect for '${moduleId}' consent refused; aspect skipped (will not re-prompt).`);
      return { consented: false, reason: 'denied' };
    }
  }
  return { consented: true };
}

/**
 * Deploy-flow glue (SC4): consulted by `module-deploy.ts` after a
 * primary deploy successfully completes. Decides whether to fan an
 * aspect out for the given trigger and dispatches if so.
 *
 * Gating logic, in order:
 *
 *   1. No `base_module_aspect` in the manifest → skip
 *      (`reason: 'no_aspect'`).
 *   2. The aspect's `triggers` list doesn't include the current
 *      trigger → skip (`reason: 'trigger_not_declared'`).
 *   3. No `aspect_approvals` row for (moduleId, version) → skip
 *      (`reason: 'no_approval'`). Surface a warning so the operator
 *      can re-import to grant consent.
 *   4. Approval exists but the manifest's scope no longer matches
 *      the approved scope_hash → skip (`reason: 'scope_changed'`).
 *      Surface a warning that re-approval is required (D7).
 *   5. Otherwise: invoke the runner with the named trigger.
 *
 * `runner` is injectable so unit tests don't need to drive real
 * Ansible — the default is `runAspectFanOut`.
 *
 * Failure semantics (per D4): a failed fan-out is reported as
 * `{ ran: true, success: false, ... }`. The PRIMARY deploy does
 * not get rolled back — aspects are forward-progress only and a
 * partial fleet update is expected to converge on the next
 * fan-out. The caller (module-deploy.ts) should log the failure
 * loudly but not change the primary deploy's success status.
 */
export async function maybeRunAspectForTrigger(args: {
  moduleId: string;
  manifest: ModuleManifest;
  trigger: BaseModuleAspectTrigger;
  db: DbClient;
  runner?: typeof runAspectFanOut;
  excludeHostnames?: string[];
  /** Injectable consent prompt (default: bus interview). See ISS-0027. */
  requestConsent?: AspectConsentRequest;
}): Promise<AspectGlueResult> {
  const { moduleId, manifest, trigger, db } = args;
  const aspect = manifest.base_module_aspect;

  if (!aspect) {
    return { ran: false, reason: 'no_aspect' };
  }

  if (!aspect.triggers.includes(trigger)) {
    return { ran: false, reason: 'trigger_not_declared' };
  }

  const moduleRow = db.select().from(modules).where(eq(modules.id, moduleId)).get();
  if (!moduleRow) {
    // The deploy flow just acted on this module — its absence
    // would be a deeper bug. Surface as "no_approval" with a log
    // because there's nothing meaningful to fan out to.
    log.warn(`Aspect glue: module '${moduleId}' not found in DB, skipping fan-out`);
    return { ran: false, reason: 'no_approval' };
  }

  const consent = await ensureAspectConsent({
    moduleId,
    version: moduleRow.version,
    aspect,
    trigger,
    db,
    requestConsent: args.requestConsent,
  });
  if (!consent.consented) {
    return { ran: false, reason: consent.reason };
  }

  const runner = args.runner ?? runAspectFanOut;
  const runResult = await runner({
    moduleId,
    aspect,
    moduleSourcePath: moduleRow.sourcePath,
    options: {
      trigger,
      excludeHostnames: args.excludeHostnames,
    },
    db,
  });
  return { ran: true, success: runResult.success, runResult };
}

/**
 * A system the inbound reconcile may apply aspects to. Deliberately just a
 * hostname and a zone: the reconcile decides eligibility from the zone and
 * hands the hostname to `planAspectFanOut`, which already knows how to reach
 * both a pool machine and a container_service LXC.
 */
export interface AspectReconcileSystem {
  hostname: string;
  /**
   * `string`, not `NetworkZone`, to match what everything on this path already
   * uses: `aspect.applicable_zones` is `string[]` from the manifest schema,
   * `getSystemsByZone` takes `string[]`, and `DeployedSystem.zone` — the
   * capability contract this is fed from — is `string`. Narrowing here would
   * only add a cast at every call site.
   */
  zone: string;
}

/** One provider's aspect, applied (or not) to the systems it covered. */
export interface AspectReconcileOutcome {
  /** The module whose aspect this is — NOT the module being deployed. */
  providerModuleId: string;
  role: string;
  /** The subset of the offered systems this aspect's zones actually cover. */
  hostnames: string[];
  ran: boolean;
  success: boolean;
  error?: string;
  /** Set when `ran === false`. */
  reason?: AspectSkipReason | 'paused' | 'not_deployed' | 'no_covered_systems';
}

export interface AspectReconcileResult {
  outcomes: AspectReconcileOutcome[];
  /** The subset that ran and failed — what a caller decides fatality on. */
  failures: AspectReconcileOutcome[];
}

/**
 * The INBOUND direction: given systems that have just come into existence,
 * apply every approved aspect the fleet already provides that covers their
 * zones.
 *
 * This is the fix for celilo#902. `maybeRunAspectForTrigger` is provider-scoped
 * — one module's aspect across the whole fleet, enumerated at that instant —
 * and nothing re-runs it, so a system provisioned later never receives an
 * aspect and nothing reports the gap.
 *
 * TWO THINGS ARE DELIBERATELY NOT CONSULTED HERE, and both are load-bearing:
 *
 *  1. `aspect.triggers` (design D2). Eligibility is `applicable_zones` plus
 *     operator approval, full stop. `triggers` declares which PROVIDER-side
 *     events cause a fleet-wide fan-out; a system joining a zone is a framework
 *     event, not a provider event, and the spec's scenario does not condition
 *     on it either ("every approved aspect whose `applicable_zones` includes
 *     that zone SHALL run on the new system"). Requiring modules to declare
 *     `on_new_system_in_zone` would change every aspect's scope hash — raising
 *     a re-approval interview on the live fleet as a side effect of a bug fix —
 *     and would make correct convergence opt-in, so the next module that forgot
 *     the declaration would reintroduce this bug quietly.
 *  2. The deploying module's own aspect. `on_install` already fans that one out
 *     across the whole fleet, so the caller passes it in `excludeModuleIds`.
 *
 * PAUSED PROVIDERS ARE SKIPPED (design D4a), and that is the documented escape
 * hatch: an inbound aspect failure is fatal to the deploy that created the
 * system, so an operator whose non-essential aspect is wedging every deploy
 * pauses its provider, deploys, and unpauses.
 *
 * Consent is checked exactly as the outbound direction checks it, through the
 * shared `ensureAspectConsent`.
 */
export async function reconcileAspectsForSystems(args: {
  systems: AspectReconcileSystem[];
  db: DbClient;
  /** Providers to skip — the deploying module, whose own aspect already ran. */
  excludeModuleIds?: string[];
  runner?: typeof runAspectFanOut;
  requestConsent?: AspectConsentRequest;
}): Promise<AspectReconcileResult> {
  const { systems, db } = args;
  const outcomes: AspectReconcileOutcome[] = [];
  if (systems.length === 0) return { outcomes, failures: [] };

  const exclude = new Set(args.excludeModuleIds ?? []);
  const runner = args.runner ?? runAspectFanOut;

  for (const moduleRow of db.select().from(modules).all()) {
    if (exclude.has(moduleRow.id)) continue;

    const manifest = moduleRow.manifestData as ModuleManifest | null;
    const aspect = manifest?.base_module_aspect;
    if (!aspect) continue;

    const base = { providerModuleId: moduleRow.id, role: aspect.ansible_role };

    // A module that never deployed has nothing to fan out FROM — its
    // ansible_vars resolve against infrastructure that does not exist.
    if (moduleRow.state !== 'INSTALLED' && moduleRow.state !== 'VERIFIED') {
      if (moduleRow.state === 'PAUSED') {
        // Reported rather than silent: pausing to get past a wedged aspect is
        // legitimate, but it must not quietly become permanent.
        log.warn(
          `Aspect '${aspect.ansible_role}' from paused module '${moduleRow.id}' NOT applied to ${systems.map((s) => s.hostname).join(', ')}. Unpause and redeploy it to converge them.`,
        );
        outcomes.push({ ...base, hostnames: [], ran: false, success: true, reason: 'paused' });
        continue;
      }
      outcomes.push({ ...base, hostnames: [], ran: false, success: true, reason: 'not_deployed' });
      continue;
    }

    const zones = new Set<string>(aspect.applicable_zones);
    const covered = systems.filter((s) => zones.has(s.zone));
    if (covered.length === 0) {
      outcomes.push({
        ...base,
        hostnames: [],
        ran: false,
        success: true,
        reason: 'no_covered_systems',
      });
      continue;
    }

    const consent = await ensureAspectConsent({
      moduleId: moduleRow.id,
      version: moduleRow.version,
      aspect,
      trigger: 'on_new_system_in_zone',
      db,
      requestConsent: args.requestConsent,
    });
    if (!consent.consented) {
      outcomes.push({
        ...base,
        hostnames: covered.map((s) => s.hostname),
        ran: false,
        success: true,
        reason: consent.reason,
      });
      continue;
    }

    const hostnames = covered.map((s) => s.hostname);
    log.info(
      `Applying aspect '${aspect.ansible_role}' from '${moduleRow.id}' to newly created system(s): ${hostnames.join(', ')}`,
    );
    const runResult = await runner({
      moduleId: moduleRow.id,
      aspect,
      moduleSourcePath: moduleRow.sourcePath,
      options: { trigger: 'on_new_system_in_zone', onlyHostnames: hostnames },
      db,
    });
    outcomes.push({
      ...base,
      hostnames,
      ran: true,
      success: runResult.success,
      error: runResult.error,
    });
  }

  return { outcomes, failures: outcomes.filter((o) => o.ran && !o.success) };
}

/**
 * How an aspect stands on one system, as MEASURED against that system.
 *
 * `unknown` is not a hedge and must not be collapsed into `applied`. Ansible's
 * check mode does not evaluate a task that cannot support it — it SKIPS it —
 * so a role built from `command:` / `shell:` tasks can finish a check run with
 * `changed=0` having never been applied to the host at all. A two-state answer
 * would report that as applied: a confidently clean verdict about an
 * unconverged system, which is the same failure this whole approach exists to
 * avoid (celilo#902 design D6). Absence of a change is not evidence of
 * convergence when nothing was assessed.
 */
export type AspectCoverageState = 'applied' | 'missing' | 'unknown' | 'unreachable';

export interface AspectCoverageFinding {
  providerModuleId: string;
  role: string;
  hostname: string;
  state: AspectCoverageState;
  /** Why, in operator-facing words. Always set for anything but 'applied'. */
  detail?: string;
}

/**
 * Ask every system the fleet's approved aspects entitle it to whether it
 * actually has them (celilo#902 design D6/D7).
 *
 * NOTHING IS STORED AND NOTHING IS READ FROM A RECORD. Both halves of the
 * question are answered from live state:
 *
 *  - "should have" is `planAspectFanOut(aspect)` — literally the code that
 *    performs the fan-out, so the entitlement set cannot disagree with it. No
 *    separate query to drift.
 *  - "does have" is the aspect evaluated against the host in check mode. A row
 *    saying "applied" would be a claim about a remote host, and celilo has been
 *    burned by trusting exactly that (celilo#626: `dns_registrations` rows held
 *    a pinned `ip` and the refresher republished a dead address over correct
 *    public DNS, with every in-fleet check green).
 *
 * This SSHes to every entitled system, so it is not cheap and must not run in a
 * default `system doctor` pass — it is gated behind `--deep`.
 *
 * Consent is checked but never REQUESTED: a read-only diagnostic must not raise
 * an interview. An aspect that is unapproved, denied, or whose scope changed is
 * simply not verified.
 */
export async function verifyAspectCoverage(args: {
  db: DbClient;
  /** Restrict to one provider — `doctor --deep <module>`. */
  onlyModuleId?: string;
  /** Injected in tests, exactly as the other entry points here do it. */
  runner?: typeof runAspectFanOut;
}): Promise<AspectCoverageFinding[]> {
  const { db } = args;
  const runner = args.runner ?? runAspectFanOut;
  const findings: AspectCoverageFinding[] = [];

  for (const moduleRow of db.select().from(modules).all()) {
    if (args.onlyModuleId && moduleRow.id !== args.onlyModuleId) continue;

    const manifest = moduleRow.manifestData as ModuleManifest | null;
    const aspect = manifest?.base_module_aspect;
    if (!aspect) continue;
    if (moduleRow.state !== 'INSTALLED' && moduleRow.state !== 'VERIFIED') {
      // A paused provider's entitled systems are still reported (D4a) — pausing
      // to get past a wedged aspect is legitimate, but it must not quietly
      // become permanent. Reported without a probe: the provider is not running
      // and re-running its role against every host would be a change, not a read.
      if (moduleRow.state === 'PAUSED') {
        const plan = await planAspectFanOut(aspect);
        for (const target of plan.targetSystems) {
          findings.push({
            providerModuleId: moduleRow.id,
            role: aspect.ansible_role,
            hostname: target.hostname,
            state: 'unknown',
            detail: `'${moduleRow.id}' is PAUSED, so its aspect is not being applied to new systems. Unpause and redeploy it to converge.`,
          });
        }
      }
      continue;
    }
    if (checkAspectApproval(moduleRow.id, moduleRow.version, aspect, db) !== 'approved') continue;

    const plan = await planAspectFanOut(aspect);
    if (plan.targetSystems.length === 0) continue;

    const result = await runner({
      moduleId: moduleRow.id,
      aspect,
      moduleSourcePath: moduleRow.sourcePath,
      options: { trigger: 'on_new_system_in_zone', check: true, noInteractive: true },
      db,
    });

    const byHost = new Map(result.recap.map((r) => [r.host, r]));
    for (const target of plan.targetSystems) {
      const base = {
        providerModuleId: moduleRow.id,
        role: aspect.ansible_role,
        hostname: target.hostname,
      };
      const recap = byHost.get(target.hostname);
      if (!recap) {
        findings.push({
          ...base,
          state: 'unknown',
          detail:
            'Ansible produced no recap line for this host, so nothing was measured. Absence of a recap is not evidence the aspect is applied.',
        });
        continue;
      }
      if (recap.unreachable > 0 || recap.failed > 0) {
        findings.push({
          ...base,
          state: 'unreachable',
          detail: 'The host could not be evaluated — it did not answer, or the role errored on it.',
        });
        continue;
      }
      if (recap.skipped > 0) {
        // THE CASE THAT MUST NOT READ AS APPLIED. See AspectCoverageState.
        findings.push({
          ...base,
          state: 'unknown',
          detail: `The role has ${recap.skipped} task(s) check mode cannot evaluate, so convergence was not measured. Prefer check-capable Ansible modules (copy, template, lineinfile, file, package, service) in aspect roles, or give a command/shell task an honest changed_when:.`,
        });
        continue;
      }
      if (recap.changed > 0) {
        findings.push({
          ...base,
          state: 'missing',
          detail: `The aspect would change ${recap.changed} thing(s) on this host, so it is not applied. Run \`celilo module deploy ${moduleRow.id}\` to converge it.`,
        });
        continue;
      }
      findings.push({ ...base, state: 'applied' });
    }
  }

  return findings;
}