#!/usr/bin/env bash
#
# Staged SELinux recovery and normalization for a live Underpost / Kubernetes host.
#
#   inspect -> validate -> normalize -> verify -> enforce
#
# Never reboots, never restarts Kubernetes, kubelet, the container runtime, the CNI or any
# workload, never touches PVs/PVCs or the contents of a data directory, and never disables
# SELinux. The only mutations it can make are:
#
#   * creating a node storage directory that a PersistentVolume references and that is missing;
#   * adding a `semanage fcontext` mapping for a discovered storage root;
#   * running `restorecon` over those roots;
#   * flipping SELinux to Enforcing, and only in the `enforce` stage, and only after `verify`
#     passes on this same run.
#
# Every mutating stage requires --apply. Without it the script reports what it would do.
#
# Rocky Linux 9 / RHEL 9.
set -euo pipefail

SHARED_TYPE='container_file_t'
# Overridable so the rollback round-trip can be exercised without writing under /var/lib.
STATE_DIR="${UNDERPOST_SELINUX_STATE_DIR:-/var/lib/underpost/selinux-normalize}"
KUBECTL_TIMEOUT='20s'
STAGE='inspect'
APPLY=0
ASSUME_YES=0
AVC_SINCE=''
AVC_SINCE_EXPLICIT=0
ALLOW_UNLABELED_MOUNTS=0
declare -a EXTRA_PATHS=()

RED=''; YELLOW=''; GREEN=''; BOLD=''; RESET=''
if [ -t 1 ]; then RED=$'\033[31m'; YELLOW=$'\033[33m'; GREEN=$'\033[32m'; BOLD=$'\033[1m'; RESET=$'\033[0m'; fi

say()  { printf '%s\n' "$*"; }
head1() { printf '\n%s==> %s%s\n' "$BOLD" "$*" "$RESET"; }
ok()   { printf '  %s[ ok ]%s %s\n' "$GREEN" "$RESET" "$*"; }
warn() { printf '  %s[warn]%s %s\n' "$YELLOW" "$RESET" "$*"; }
bad()  { printf '  %s[fail]%s %s\n' "$RED" "$RESET" "$*"; }
info() { printf '  %s\n' "$*"; }

usage() {
    cat <<'USAGE'
Usage: selinux-normalize.sh [STAGE] [options]

Stages (each one re-runs the ones before it that are read-only):
  inspect     Report SELinux state, Kubernetes state, discovered storage and current labels.
  validate    inspect + assert every precondition for a safe normalization. Read-only.
  normalize   Register the persistent container_file_t mappings and restore the discovered
              storage roots. Requires --apply to mutate.
  verify      Re-check labels, workload health and fresh AVC denials since the last normalize.
  enforce     Transition Permissive -> Enforcing, only if verify passes. Requires --apply.
  rollback    Undo the fcontext mappings this script added, and restore the affected trees.
              Requires --apply.
  mark        Move the denial window to now, without touching labels or SELinux mode. Use after
              suspending or fixing a workload, so verify measures the state you just created
              rather than the denials that led you to change it. Requires --apply.

Options:
  --apply             Perform mutations. Without it, mutating stages only report.
  --yes               Do not prompt before the Enforcing transition.
  --allow-unlabeled-mounts
                      Let verify pass with workload hostPath mounts still unresolved. Those
                      mounts deny under Enforcing; only pass this once you have decided each
                      one is acceptable to break.
  --path PATH         Additional node storage root to include (repeatable).
  --since WHEN        ausearch window for the AVC scan (default: recent).
  -h, --help          This text.

Exit codes: 0 success, 1 stage failed / unsafe to proceed, 2 usage or missing tooling.
USAGE
}

while [ "$#" -gt 0 ]; do
    case "$1" in
        inspect|validate|normalize|verify|enforce|rollback|mark) STAGE="$1"; shift ;;
        --apply) APPLY=1; shift ;;
        --yes) ASSUME_YES=1; shift ;;
        --allow-unlabeled-mounts) ALLOW_UNLABELED_MOUNTS=1; shift ;;
        --path) [ "$#" -ge 2 ] || { echo "--path requires a value" >&2; exit 2; }; EXTRA_PATHS+=("$2"); shift 2 ;;
        --since) [ "$#" -ge 2 ] || { echo "--since requires a value" >&2; exit 2; }; AVC_SINCE="$2"; AVC_SINCE_EXPLICIT=1; shift 2 ;;
        -h|--help) usage; exit 0 ;;
        *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
    esac
done

SUDO=''
SUDO_RO=''
if [ "$(id -u)" -ne 0 ]; then
    SUDO='sudo'
    # Read-only probes never prompt: an inspect/validate/verify run must not block on a password,
    # and a probe that cannot elevate degrades to "unknown" rather than hanging the operator.
    SUDO_RO='sudo -n'
fi

privileged_read() {
    if [ -z "$SUDO_RO" ]; then "$@"; else $SUDO_RO "$@"; fi
}

run() {
    if [ "$APPLY" -eq 1 ]; then
        info "+ $*"
        eval "$@"
    else
        info "would run: $*"
    fi
}

# ---------------------------------------------------------------------------- inspect: SELinux

SELINUX_ENABLED=0
SELINUX_MODE='unknown'
SELINUX_CONFIG_MODE='unknown'
SELINUX_POLICY='unknown'
HAVE_SEMANAGE=0
HAVE_RESTORECON=0
HAVE_AUSEARCH=0

inspect_selinux() {
    head1 'SELinux state'
    if ! command -v getenforce >/dev/null 2>&1; then
        bad 'getenforce is missing: SELinux userspace is not installed.'
        info 'Remediation: dnf install -y policycoreutils policycoreutils-python-utils selinux-policy-targeted audit'
        return
    fi
    SELINUX_MODE="$(getenforce)"
    command -v selinuxenabled >/dev/null 2>&1 && selinuxenabled && SELINUX_ENABLED=1
    [ -r /etc/selinux/config ] && SELINUX_CONFIG_MODE="$(awk -F= '/^SELINUX=/{print $2}' /etc/selinux/config)"
    [ -r /etc/selinux/config ] && SELINUX_POLICY="$(awk -F= '/^SELINUXTYPE=/{print $2}' /etc/selinux/config)"
    command -v semanage    >/dev/null 2>&1 && HAVE_SEMANAGE=1
    command -v restorecon  >/dev/null 2>&1 && HAVE_RESTORECON=1
    command -v ausearch    >/dev/null 2>&1 && HAVE_AUSEARCH=1

    info "runtime mode      : $SELINUX_MODE"
    info "config mode       : $SELINUX_CONFIG_MODE"
    info "policy            : $SELINUX_POLICY"
    info "semanage present  : $([ "$HAVE_SEMANAGE" -eq 1 ] && echo yes || echo 'no  <-- policycoreutils-python-utils')"
    info "restorecon present: $([ "$HAVE_RESTORECON" -eq 1 ] && echo yes || echo 'no  <-- policycoreutils')"
    info "ausearch present  : $([ "$HAVE_AUSEARCH" -eq 1 ] && echo yes || echo 'no  <-- audit')"
    [ -e /.autorelabel ] && warn '/.autorelabel is present: the next boot will relabel the whole filesystem.'
    return 0
}

# ------------------------------------------------------------------------- inspect: Kubernetes

KUBE_AVAILABLE=0
NODES_TOTAL=0
NODES_READY=0
declare -a STORAGE_PATHS=()
declare -a MANAGED_ROOTS=()
declare -a PV_SUBTREES=()
declare -a ROOT_REASON=()
declare -a REFUSED=()
declare -a UNCOVERED=()
declare -a POD_HOSTPATHS=()
declare -a RESIDUAL=()
declare -a AUDIT_SINCE_ARGS=(-ts recent)

kubectl_q() { kubectl --request-timeout="$KUBECTL_TIMEOUT" "$@" 2>/dev/null; }

inspect_kubernetes() {
    head1 'Kubernetes state'
    if ! command -v kubectl >/dev/null 2>&1; then
        warn 'kubectl is not on PATH; storage discovery falls back to the declared platform roots.'
        return 0
    fi
    if ! kubectl_q version -o json >/dev/null; then
        warn 'No reachable API server; storage discovery falls back to the declared platform roots.'
        return 0
    fi
    KUBE_AVAILABLE=1
    local nodes
    nodes="$(kubectl_q get nodes --no-headers || true)"
    NODES_TOTAL="$(printf '%s' "$nodes" | grep -c . || true)"
    NODES_READY="$(printf '%s' "$nodes" | awk '$2 ~ /(^|,)Ready(,|$)/' | grep -c . || true)"
    info "nodes ready       : ${NODES_READY}/${NODES_TOTAL}"
    info "this host         : $(hostname)"
    return 0
}

# Paths that must never receive the shared container label, whatever discovery turns up.
#
# A pod hostPath is not evidence that a directory is workload data: the CNI plugin directory, the
# container runtime socket, /proc and the kubelet tree are all mounted into pods and every one of
# them has a policy type that exists for a reason. Relabeling /opt/cni/bin would strip `bin_t`
# from the calico executables the kubelet runs.
PROTECTED_PATHS_REGEX='^(/|/usr|/etc|/bin|/sbin|/lib|/lib64|/boot|/proc|/sys|/dev|/run|/var|/var/lib|/var/run|/var/log|/var/lib/kubelet|/var/lib/containerd|/var/lib/docker|/var/lib/etcd|/var/lib/calico|/var/lib/cni|/opt|/opt/cni|/opt/cni/bin|/home|/root|/srv|/mnt|/tmp)$'

# Types whose whole purpose is something other than container data. `usr_t` is deliberately absent:
# it is the policy default under /opt and is exactly what the local-path provisioner root wrongly
# carries, so refusing it would refuse the fix.
PROTECTED_TYPES='bin_t lib_t shell_exec_t etc_t kubernetes_file_t container_runtime_exec_t container_var_lib_t cni_var_lib_t systemd_unit_file_t passwd_file_t shadow_t admin_home_t'

# The node directory backing every hostPath PersistentVolume the deploy flow materializes. Read
# from the source of truth the CLI itself uses, so the script and the code can never disagree.
resolve_host_volume_root() {
    local repo_env script_dir
    script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
    repo_env="$script_dir/../src/server/runtime/environment.js"
    if [ -r "$repo_env" ]; then
        local value
        value="$(sed -nE "s/^const HOST_VOLUME_ROOT = '([^']+)';.*/\1/p" "$repo_env" | head -1)"
        [ -n "$value" ] && { printf '%s' "$value"; return 0; }
    fi
    printf '%s' "${UNDERPOST_HOST_VOLUME_ROOT:-/home/dd/engine/volume}"
}

is_protected_path() {
    printf '%s' "$1" | grep -qE "$PROTECTED_PATHS_REGEX"
}

is_protected_type() {
    local type; type="$(path_type "$1")"
    [ -n "$type" ] || return 1
    printf '%s' " $PROTECTED_TYPES " | grep -q " $type "
}

# Classifies a pod hostPath mount so the report separates the mounts that need an operator
# decision from the ones whose current type is already correct.
#
#   system  the path is inside a protected tree, or already carries a type the container policy
#           defines for that role (`container_var_lib_t`, `cert_t`, `proc_t`, `kubernetes_file_t`
#           …). These are mounted by privileged system DaemonSets and are not workload data.
#   ok      already container-accessible.
#   review  neither — the only class worth acting on.
classify_pod_mount() {
    local path="$1" type="$2"
    [ -e "$path" ] || { printf 'absent'; return 0; }
    [ "$type" = "$SHARED_TYPE" ] && { printf 'ok'; return 0; }
    is_protected_path "$path" && { printf 'system'; return 0; }
    case "$type" in
        container_*|proc_t|sysfs_t|cert_t|kubernetes_file_t|modules_object_t|iptables_var_run_t|var_run_t|var_lib_t|root_t|etc_t|bin_t|lib_t|shell_exec_t)
            printf 'system'; return 0 ;;
    esac
    printf 'review'
}

# Storage discovery, in three tiers with different authority:
#
#   managed roots   mapped with `semanage fcontext` and relabeled recursively. Only directories
#                   this platform owns end up here: its declared data roots, the local-path
#                   provisioner's configured node roots, and anything the operator named.
#   PV subtrees     node-local paths a PersistentVolume actually references. Used to prove the
#                   managed roots cover the live storage; one that is not covered is reported
#                   and skipped, never silently relabeled or promoted to a root of its own.
#   pod hostPaths   reported only. A pod mounting a host directory says nothing about whether
#                   that directory is workload data.
discover_pv_paths() {
    head1 'PersistentVolume / PersistentVolumeClaim storage discovery'
    local host_volume_root; host_volume_root="$(resolve_host_volume_root)"

    if [ "$KUBE_AVAILABLE" -eq 1 ]; then
        local rows
        rows="$(kubectl_q get pv -o custom-columns=\
NAME:.metadata.name,CLASS:.spec.storageClassName,PHASE:.status.phase,\
CLAIM:.spec.claimRef.name,HOSTPATH:.spec.hostPath.path,LOCALPATH:.spec.local.path || true)"
        if [ -n "$rows" ]; then
            printf '%s\n' "$rows" | sed 's/^/  /'
            local name class phase claim hostpath localpath path
            while read -r name class phase claim hostpath localpath; do
                [ -n "$name" ] && [ "$name" != 'NAME' ] || continue
                path=''
                [ "$hostpath" != '<none>' ] && path="$hostpath"
                [ -z "$path" ] && [ "$localpath" != '<none>' ] && path="$localpath"
                [ -n "$path" ] && PV_SUBTREES+=("$path") || true
            done <<< "$rows"
        else
            info 'no PersistentVolumes found'
        fi

        # Dynamically provisioned claims have no path of their own: the data is a subdirectory of
        # the provisioner's node root, so the root is what must carry the label. Read it from the
        # live ConfigMap rather than assuming the upstream default.
        local ns cfg roots root
        for ns in local-path-storage kube-system; do
            cfg="$(kubectl_q get configmap local-path-config -n "$ns" -o jsonpath='{.data.config\.json}' || true)"
            [ -n "$cfg" ] || continue
            roots="$(printf '%s' "$cfg" | tr -d ' \n' | grep -oE '"paths":\[[^]]*\]|"sharedFileSystemPath":"[^"]*"' | grep -oE '/[^",]+' || true)"
            if [ -n "$roots" ]; then
                say ''
                info "local-path provisioner (ns/$ns) node roots:"
                while read -r root; do
                    [ -n "$root" ] || continue
                    info "  $root"
                    MANAGED_ROOTS+=("$root")
                    ROOT_REASON+=("$root|local-path provisioner node root (ns/$ns)")
                done <<< "$roots"
            fi
            break
        done

        # Reported, never labeled. A host directory being mounted into a pod says nothing about
        # whether it holds workload data, so the decision is left to the operator — but each mount
        # is classified so the handful that actually need a decision are not buried under the
        # couple of dozen host-integration mounts every CNI and kubelet DaemonSet carries.
        #
        # `go-template` rather than `jsonpath`: inside a nested range, jsonpath's `$` resolves to
        # the document root, not the enclosing item, so the pod's own namespace and name came back
        # empty and every row read as a bare "/".
        local pod_rows p pods current class width
        #
        # Three sources, not one. A pod only exists while it runs, so reading pods alone reports a
        # clean host for a CronJob that is merely scheduled — its mount reappears at the next fire.
        # The workload templates are the durable declaration and are what must be audited.
        # Terminal pods are excluded for the mirror-image reason: a finished Job cannot produce
        # another denial, and its own template already speaks for the next run.
        pod_rows="$(
            {
                kubectl_q get pods -A --field-selector=status.phase!=Succeeded,status.phase!=Failed \
                    -o go-template='{{range .items}}{{$ns := .metadata.namespace}}{{$n := .metadata.name}}{{range .spec.volumes}}{{if .hostPath}}{{.hostPath.path}}{{"\t"}}{{$ns}}/{{$n}}{{"\n"}}{{end}}{{end}}{{end}}' || true
                kubectl_q get deployments,statefulsets,daemonsets,jobs,replicasets -A \
                    -o go-template='{{range .items}}{{$ns := .metadata.namespace}}{{$k := .kind}}{{$n := .metadata.name}}{{range .spec.template.spec.volumes}}{{if .hostPath}}{{.hostPath.path}}{{"\t"}}{{$ns}}/{{$k}}/{{$n}}{{"\n"}}{{end}}{{end}}{{end}}' || true
                kubectl_q get cronjobs -A \
                    -o go-template='{{range .items}}{{$ns := .metadata.namespace}}{{$n := .metadata.name}}{{if not .spec.suspend}}{{range .spec.jobTemplate.spec.template.spec.volumes}}{{if .hostPath}}{{.hostPath.path}}{{"\t"}}{{$ns}}/CronJob/{{$n}}{{"\n"}}{{end}}{{end}}{{end}}{{end}}' || true
            } | grep -E '^/' | sort -u || true
        )"
        if [ -n "$pod_rows" ]; then
            say ''
            info 'hostPath mounts declared by live pods and workload templates (never relabeled here):'
            width="$(printf '%s\n' "$pod_rows" | cut -f1 | awk '{ if (length($0) > m) m = length($0) } END { print (m > 58 ? 58 : (m < 24 ? 24 : m)) }')"
            printf "  %-${width}s %-22s %-8s %s\n" PATH 'CURRENT TYPE' CLASS 'MOUNTED BY'
            while read -r p; do
                [ -n "$p" ] || continue
                pods="$(printf '%s\n' "$pod_rows" | awk -F'\t' -v k="$p" '$1 == k {print $2}' | paste -sd, - | cut -c1-56)"
                current="$(path_type "$p")"
                class="$(classify_pod_mount "$p" "$current")"
                printf "  %-${width}s %-22s %-8s %s\n" "$p" "${current:-absent}" "$class" "$pods"
                [ "$class" = 'review' ] && POD_HOSTPATHS+=("$p") || true
            done <<< "$(printf '%s\n' "$pod_rows" | cut -f1 | sort -u)"
            info 'system  = host-integration mount (CNI, kubelet, runtime, /proc); its type is correct as-is.'
            info 'ok      = already container-accessible.'
            info 'absent  = not present on this node (a mount belonging to a pod scheduled elsewhere).'
            info 'review  = none of the above, and outside every managed root — the rows needing a decision.'
        fi
    fi

    # Declared platform roots. Kept even when the API is reachable: a root can exist on this node
    # with no object currently referencing it (a Retained volume between deploys).
    MANAGED_ROOTS+=('/data')
    ROOT_REASON+=('/data|declared platform data root')
    MANAGED_ROOTS+=("$host_volume_root")
    ROOT_REASON+=("$host_volume_root|declared hostPath volume root (HOST_VOLUME_ROOT)")
    if [ "$KUBE_AVAILABLE" -ne 1 ]; then
        MANAGED_ROOTS+=('/opt/local-path-provisioner')
        ROOT_REASON+=('/opt/local-path-provisioner|local-path provisioner default root (no API to query)')
    fi
    local extra
    for extra in ${EXTRA_PATHS[@]+"${EXTRA_PATHS[@]}"}; do
        MANAGED_ROOTS+=("$extra")
        ROOT_REASON+=("$extra|named by the operator with --path")
    done

    # Collapse to non-overlapping roots: an fcontext expression is `<root>(/.*)?`, so a nested
    # entry is a redundant rule and a second relabel pass over the same tree. Only managed roots
    # take part — a discovered subtree can never absorb or widen one.
    local sorted candidate covered root
    sorted="$(printf '%s\n' "${MANAGED_ROOTS[@]}" | sed 's:/*$::' | grep -E '^/.+' | sort -u)"
    MANAGED_ROOTS=()
    while read -r candidate; do
        [ -n "$candidate" ] || continue
        covered=0
        while read -r root; do
            { [ -n "$root" ] && [ "$root" != "$candidate" ] && case "$candidate" in "$root"/*) covered=1 ;; esac; } || true
        done <<< "$sorted"
        [ "$covered" -eq 0 ] && MANAGED_ROOTS+=("$candidate") || true
    done <<< "$sorted"

    # Refuse anything whose type or path says it is not workload data, before it can be reported
    # as an intended change. A refusal is fatal in `validate`, so it can never reach `--apply`.
    STORAGE_PATHS=()
    for candidate in "${MANAGED_ROOTS[@]}"; do
        if is_protected_path "$candidate"; then
            REFUSED+=("$candidate|system directory; relabeling it would change the policy type of host-owned files")
            continue
        fi
        if is_protected_type "$candidate"; then
            REFUSED+=("$candidate|carries the protected type '$(path_type "$candidate")'; that type is not container data")
            continue
        fi
        STORAGE_PATHS+=("$candidate")
    done

    head1 'Storage roots selected for labeling'
    if [ "${#STORAGE_PATHS[@]}" -eq 0 ]; then
        warn 'none'
    else
        local reason entry
        for candidate in "${STORAGE_PATHS[@]}"; do
            reason=''
            for entry in ${ROOT_REASON[@]+"${ROOT_REASON[@]}"}; do
                [ "${entry%%|*}" = "$candidate" ] && reason="${entry#*|}"
            done
            printf '  %-42s %s\n' "$candidate" "${reason:-discovered}"
        done
    fi

    if [ "${#REFUSED[@]}" -gt 0 ]; then
        head1 'Refused (never relabeled)'
        for candidate in "${REFUSED[@]}"; do
            printf '  %-42s %s\n' "${candidate%%|*}" "${candidate#*|}"
        done
    fi

    # Every live PersistentVolume path must fall inside a managed root, otherwise its data is not
    # actually being fixed by this run and the operator has to say so explicitly.
    UNCOVERED=()
    local subtree
    for subtree in ${PV_SUBTREES[@]+"${PV_SUBTREES[@]}"}; do
        covered=0
        for root in ${STORAGE_PATHS[@]+"${STORAGE_PATHS[@]}"}; do
            { [ "$subtree" = "$root" ] || case "$subtree" in "$root"/*) covered=1 ;; esac; } || true
            [ "$subtree" = "$root" ] && covered=1 || true
        done
        [ "$covered" -eq 0 ] && UNCOVERED+=("$subtree") || true
    done
    if [ "${#UNCOVERED[@]}" -gt 0 ]; then
        head1 'PersistentVolume paths NOT covered by any managed root'
        printf '%s\n' "${UNCOVERED[@]}" | sort -u | sed 's/^/  /'
        warn 'These are not labeled by this run. Add one with --path if it is genuinely workload data.'
    fi

    # POD_HOSTPATHS now holds only the mounts classified `review`: not a system path, not already
    # container-accessible. Those still inside a managed root are fine — the relabel covers them.
    # Whatever is left is the complete set of places a denial can still come from after this run.
    local mount current
    RESIDUAL=()
    for mount in ${POD_HOSTPATHS[@]+"${POD_HOSTPATHS[@]}"}; do
        [ -e "$mount" ] || continue
        covered=0
        for root in ${STORAGE_PATHS[@]+"${STORAGE_PATHS[@]}"}; do
            { [ "$mount" = "$root" ] && covered=1; } || case "$mount" in "$root"/*) covered=1 ;; esac
        done
        [ "$covered" -eq 0 ] && RESIDUAL+=("$mount|$(path_type "$mount")") || true
    done
    if [ "${#RESIDUAL[@]}" -gt 0 ]; then
        head1 'Workload hostPath mounts this run does not fix'
        for mount in "${RESIDUAL[@]}"; do
            printf '  %-52s %s\n' "${mount%%|*}" "${mount#*|}"
        done
        info 'Deliberate exclusions, and the only remaining source of a container denial. If a'
        info '  workload genuinely needs to read or write one, move that data under a storage root,'
        info '  or name the specific subdirectory with --path — never the whole tree.'
    fi
    return 0
}

# Prints the SELinux type of a path, or nothing when it cannot be read.
#
# Never fails. `set -o pipefail` makes `ls -Zd <missing> | awk ...` a failing pipeline, and a
# `current="$(path_type "$p")"` assignment takes that status — which under `set -e` aborted the
# whole run on the first path that does not exist on this node (a mount belonging to a pod on
# another node, or a PV pinned elsewhere).
path_type() {
    [ -e "$1" ] || return 0
    { ls -Zd "$1" 2>/dev/null || true; } | awk '{print $1}' | awk -F: '{print $3}' || true
}

report_labels() {
    head1 'Current labels vs intended'
    printf '  %-46s %-22s %-22s %s\n' PATH CURRENT INTENDED ACTION
    local path current mismatched=0 missing=0
    for path in "${STORAGE_PATHS[@]}"; do
        if [ ! -e "$path" ]; then
            printf '  %-46s %-22s %-22s %s\n' "$path" '(absent)' "$SHARED_TYPE" 'create + label'
            missing=$((missing + 1))
            continue
        fi
        current="$(path_type "$path")"
        if [ "$current" = "$SHARED_TYPE" ]; then
            printf '  %-46s %-22s %-22s %s\n' "$path" "$current" "$SHARED_TYPE" 'none'
        else
            printf '  %-46s %-22s %-22s %s\n' "$path" "${current:-unknown}" "$SHARED_TYPE" 'relabel'
            mismatched=$((mismatched + 1))
        fi
    done
    # Nested entries carrying a stale label under an already-correct root: `restorecon -R` fixes
    # them, and this is what proves the recursion is actually needed.
    local stale
    for path in "${STORAGE_PATHS[@]}"; do
        [ -d "$path" ] || continue
        stale="$(privileged_read find "$path" -mindepth 1 -maxdepth 3 -exec ls -Zd {} + 2>/dev/null \
            | awk -F: -v t="$SHARED_TYPE" '$3 != t' | head -5 || true)"
        [ -n "$stale" ] && { warn "entries under $path carrying a non-$SHARED_TYPE type:"; printf '%s\n' "$stale" | sed 's/^/      /'; }
    done
    MISMATCHED_COUNT="$mismatched"
    MISSING_COUNT="$missing"
    return 0
}

# Timestamp of the last applied normalization, in ausearch's own format.
normalize_marker_file() { printf '%s/last-normalize.txt' "$STATE_DIR"; }

# Resolves the audit window into ausearch arguments.
#
# Defaults to the moment the last `normalize --apply` finished rather than to `recent`: the
# denials this run just fixed are still inside a ten-minute `recent` window, so verifying against
# it would count the very records the relabel eliminated and never report a clean host. An
# explicit `--since` always wins.
resolve_avc_window() {
    AUDIT_SINCE_ARGS=()
    local marker; marker="$(normalize_marker_file)"
    if [ "$AVC_SINCE_EXPLICIT" -eq 0 ] && [ -r "$marker" ]; then
        local stamp; stamp="$(head -1 "$marker")"
        if [ -n "$stamp" ]; then
            # shellcheck disable=SC2206 -- deliberate split: ausearch takes date and time as two args.
            AUDIT_SINCE_ARGS=(-ts $stamp)
            AVC_SINCE="$stamp (last applied normalization)"
            return 0
        fi
    fi
    [ -n "$AVC_SINCE" ] || AVC_SINCE='recent'
    AUDIT_SINCE_ARGS=(-ts "$AVC_SINCE")
    return 0
}

report_avc() {
    resolve_avc_window
    head1 "AVC denials since: $AVC_SINCE"
    if [ "$HAVE_AUSEARCH" -ne 1 ]; then
        warn 'ausearch is unavailable; install the audit package to see denials.'
        AVC_COUNT=-1
        return 0
    fi
    local raw
    if ! privileged_read ausearch -m AVC,USER_AVC,SELINUX_ERR "${AUDIT_SINCE_ARGS[@]}" --raw >/dev/null 2>&1; then
        if [ "$(id -u)" -ne 0 ] && ! sudo -n true 2>/dev/null; then
            warn 'the audit log needs root and passwordless sudo is unavailable; re-run this stage as root.'
            AVC_COUNT=-1
            return 0
        fi
    fi
    raw="$(privileged_read ausearch -m AVC,USER_AVC,SELINUX_ERR "${AUDIT_SINCE_ARGS[@]}" --raw 2>/dev/null || true)"
    AVC_COUNT="$(printf '%s' "$raw" | grep -c 'avc:  denied' || true)"
    if [ "$AVC_COUNT" -eq 0 ]; then
        ok 'no denials in the window'
        return 0
    fi
    warn "$AVC_COUNT denial record(s); grouped by source domain, target type and target class:"
    local denials
    denials="$(printf '%s' "$raw" | grep 'avc:  denied' || true)"
    printf '%s' "$denials" \
        | sed -E 's/.*comm="?([^" ]+)"?.*scontext=([^ ]+).*tcontext=([^ ]+).*tclass=([^ ]+).*/\1 \2 \3 \4/' \
        | awk '{ sub(/:c[0-9,c]+$/, "", $2); print }' \
        | sort | uniq -c | sort -rn | head -20 | sed 's/^/      /'

    # Not every denial is a labeling problem, and telling an operator to relabel something that
    # carries no file label at all sends them after the wrong fix. A denial whose target class is
    # a filesystem object is answered by `normalize`; one whose target is another domain — MCS
    # category separation on a keyring, a socket, a process — is not, and stays after any relabel.
    AVC_FILE_COUNT="$(printf '%s' "$denials" | grep -cE 'tclass=(file|dir|lnk_file|chr_file|blk_file|fifo_file|sock_file)' || true)"
    AVC_OTHER_COUNT=$((AVC_COUNT - AVC_FILE_COUNT))
    say ''
    info "filesystem-object denials : $AVC_FILE_COUNT  (resolved by labeling — see the mount report above)"
    info "other-class denials       : $AVC_OTHER_COUNT  (NOT resolved by labeling)"
    if [ "$AVC_OTHER_COUNT" -gt 0 ]; then
        printf '%s' "$denials" \
            | grep -vE 'tclass=(file|dir|lnk_file|chr_file|blk_file|fifo_file|sock_file)' \
            | sed -E 's/.*comm="?([^" ]+)"?.*tclass=([^ ]+).*/\1 tclass=\2/' \
            | sort | uniq -c | sort -rn | head -10 | sed 's/^/      /'
        info 'Relabeling will not clear these. Confirm each is tolerated by the workload before'
        info '  enforcing — a keyring or socket probe an application already ignores is harmless,'
        info '  a denial it depends on is not.'
    fi
    return 0
}

# ---------------------------------------------------------------------------------- validate

validate() {
    head1 'Preconditions'
    local failures=0

    if [ "$SELINUX_ENABLED" -ne 1 ]; then
        bad 'SELinux is disabled. A relabel is meaningless until it is enabled and the filesystem'
        info '  has been relabeled at boot. Set SELINUX=permissive in /etc/selinux/config, touch'
        info '  /.autorelabel, and reboot during a maintenance window before running this script.'
        failures=$((failures + 1))
    else
        ok "SELinux enabled, mode $SELINUX_MODE, policy $SELINUX_POLICY"
    fi

    if [ "$SELINUX_POLICY" != 'targeted' ]; then
        bad "policy is '$SELINUX_POLICY'; this script only reasons about the targeted policy."
        failures=$((failures + 1))
    fi

    if [ "$HAVE_SEMANAGE" -ne 1 ]; then
        bad 'semanage is missing. Persistent mappings cannot be registered, and chcon alone would'
        info '  be undone by the next relabel. Remediation:'
        info "    $SUDO dnf install -y policycoreutils-python-utils"
        failures=$((failures + 1))
    else
        ok 'semanage available'
    fi

    if [ "$HAVE_RESTORECON" -ne 1 ]; then
        bad "restorecon is missing. Remediation: $SUDO dnf install -y policycoreutils"
        failures=$((failures + 1))
    else
        ok 'restorecon available'
    fi

    if [ "$KUBE_AVAILABLE" -eq 1 ]; then
        if [ "$NODES_TOTAL" -gt 0 ] && [ "$NODES_READY" -eq "$NODES_TOTAL" ]; then
            ok "all $NODES_TOTAL node(s) Ready"
        else
            warn "$NODES_READY/$NODES_TOTAL nodes Ready — normalization is still safe (it touches no"
            info '  workload), but do not proceed to the Enforcing stage on a degraded cluster.'
        fi
        local notrunning
        notrunning="$(kubectl_q get pods -A --no-headers | awk '$4 !~ /^(Running|Completed|Succeeded)$/' | grep -c . || true)"
        [ "$notrunning" -eq 0 ] && ok 'every pod is Running/Completed' \
            || warn "$notrunning pod(s) are neither Running nor Completed"
    else
        warn 'Kubernetes is unreachable; discovery used the declared platform roots only.'
    fi

    if [ "${#STORAGE_PATHS[@]}" -eq 0 ]; then
        bad 'discovery selected no storage root; there is nothing this run could normalize.'
        failures=$((failures + 1))
    fi

    # Belt and braces: the same guard discovery already applied, re-checked here so a future
    # change to discovery cannot route around it on the way to `--apply`.
    local path
    for path in ${STORAGE_PATHS[@]+"${STORAGE_PATHS[@]}"}; do
        if is_protected_path "$path"; then
            bad "refusing to operate on system directory '$path'"
            failures=$((failures + 1))
        elif is_protected_type "$path"; then
            bad "refusing to relabel '$path': it carries the protected type '$(path_type "$path")'"
            failures=$((failures + 1))
        fi
    done

    if [ "${#REFUSED[@]}" -gt 0 ]; then
        ok "${#REFUSED[@]} discovered path(s) excluded by the protection rules (listed above)"
    fi
    if [ "${#UNCOVERED[@]}" -gt 0 ]; then
        warn "${#UNCOVERED[@]} PersistentVolume path(s) fall outside every managed root and will not be"
        info '  labeled by this run. That is correct for a volume pinned to another node; for one on'
        info '  this node it means the fix is incomplete — add it with --path.'
    fi

    if [ "$failures" -eq 0 ]; then
        ok 'all preconditions met'
        return 0
    fi
    bad "$failures precondition(s) failed; no mutation will be attempted."
    return 1
}

# --------------------------------------------------------------------------------- normalize

state_file() { printf '%s/fcontext-added.txt' "$STATE_DIR"; }

record_state() {
    [ "$APPLY" -eq 1 ] || return 0
    $SUDO install -d -m 0750 "$STATE_DIR"
    printf '%s\n' "$1" | $SUDO tee -a "$(state_file)" >/dev/null
}

fcontext_registered() {
    privileged_read semanage fcontext -l 2>/dev/null | grep -qE "^$(printf '%s' "$1" | sed 's/[][\\.^$*+?(){}|]/\\&/g')\(/\.\*\)\? " || return 1
}

normalize() {
    head1 'Normalizing labels'
    [ "$APPLY" -eq 1 ] || warn 'dry run: pass --apply to perform these changes'
    local path expr
    for path in "${STORAGE_PATHS[@]}"; do
        expr="${path}(/.*)?"
        if [ ! -e "$path" ]; then
            # Only ever created, never re-moded: an existing tree keeps the ownership and mode the
            # workload's own init container set.
            run "$SUDO install -d -m 0755 '$path'"
        fi
        if fcontext_registered "$path"; then
            ok "mapping already registered: $expr -> $SHARED_TYPE"
        else
            # `-a` falls back to `-m`: some semanage versions treat an entry that already exists
            # as an error rather than a warning, and under `set -e` that would abort the run
            # partway through, leaving some roots relabeled and others not.
            run "$SUDO semanage fcontext -a -t $SHARED_TYPE '$expr' 2>/dev/null || $SUDO semanage fcontext -m -t $SHARED_TYPE '$expr'"
            # Recorded only when the mapping was absent beforehand, so a rollback never deletes an
            # entry that a previous `underpost cluster` run legitimately owns.
            record_state "$path"
        fi
    done
    for path in "${STORAGE_PATHS[@]}"; do
        [ -e "$path" ] || continue
        # -R descends into the volume trees; -F resets the SELinux user as well, because entries
        # created by the CLI carry unconfined_u where the runtime's carry system_u. Contents,
        # ownership, mode, ACLs and other xattrs are untouched.
        #
        # `|| true` on the pipeline: restorecon exits non-zero when it cannot relabel a single
        # entry (a file deleted mid-walk by a running database is routine), and one such entry
        # must not abandon the remaining roots. A leftover mislabel is caught by `verify`.
        run "{ $SUDO restorecon -RFv '$path' || true; } | tail -40"
    done
    if [ "$APPLY" -eq 1 ]; then
        # Written last: `verify` measures denials from here, so it must mark the point at which
        # every label was already correct.
        $SUDO install -d -m 0750 "$STATE_DIR"
        date '+%m/%d/%Y %H:%M:%S' | $SUDO tee "$(normalize_marker_file)" >/dev/null
        ok 'normalization applied'
        info "verify will count denials from $(cat "$(normalize_marker_file)" 2>/dev/null || echo now)"
    else
        warn 'nothing was changed'
    fi
    return 0
}

# ------------------------------------------------------------------------------------ verify

verify() {
    head1 'Verification'
    local failures=0

    report_labels
    if [ "${MISMATCHED_COUNT:-0}" -eq 0 ] && [ "${MISSING_COUNT:-0}" -eq 0 ]; then
        ok 'every discovered storage root carries the shared container type'
    else
        bad "${MISMATCHED_COUNT:-0} root(s) mislabeled, ${MISSING_COUNT:-0} absent"
        failures=$((failures + 1))
    fi

    if [ "$KUBE_AVAILABLE" -eq 1 ]; then
        head1 'Workload health'
        local bound unbound
        bound="$(kubectl_q get pvc -A --no-headers | awk '$3 == "Bound"' | grep -c . || true)"
        unbound="$(kubectl_q get pvc -A --no-headers | awk '$3 != "Bound"' | grep -c . || true)"
        info "PVCs bound: $bound, not bound: $unbound"
        [ "$unbound" -eq 0 ] || { bad 'some PVCs are not Bound'; failures=$((failures + 1)); }

        local unhealthy
        unhealthy="$(kubectl_q get pods -A --no-headers | awk '$4 !~ /^(Running|Completed|Succeeded)$/' || true)"
        if [ -n "$unhealthy" ]; then
            bad 'pods not in a healthy phase:'
            printf '%s\n' "$unhealthy" | sed 's/^/      /'
            failures=$((failures + 1))
        else
            ok 'every pod is Running/Completed'
        fi

        local restarts
        restarts="$(kubectl_q get pods -A --no-headers | awk '$5 > 0 {print "      " $0}' | head -10 || true)"
        [ -n "$restarts" ] && { warn 'pods with restarts (check whether they predate this run):'; printf '%s\n' "$restarts"; }
    else
        warn 'Kubernetes unreachable; workload health could not be verified.'
        failures=$((failures + 1))
    fi

    # A scheduled workload declaring a mount no container can use is a denial that has not
    # happened yet, not an absent one — and it is invisible in the audit log until that workload
    # next runs. Failing here is what keeps `verify` from blessing a host whose nightly CronJob
    # breaks the moment Enforcing is active.
    if [ "${#RESIDUAL[@]}" -gt 0 ]; then
        if [ "$ALLOW_UNLABELED_MOUNTS" -eq 1 ]; then
            warn "${#RESIDUAL[@]} unresolved workload hostPath mount(s), accepted via --allow-unlabeled-mounts"
        else
            bad "${#RESIDUAL[@]} workload hostPath mount(s) are still unresolved (listed above). Each one"
            info '  denies under Enforcing the next time its workload runs, whether or not a pod exists now.'
            info '  Resolve them, or re-run with --allow-unlabeled-mounts to accept the breakage knowingly.'
            failures=$((failures + 1))
        fi
    fi

    local uncovered
    for uncovered in ${UNCOVERED[@]+"${UNCOVERED[@]}"}; do
        [ -e "$uncovered" ] || continue
        if [ "$(path_type "$uncovered")" != "$SHARED_TYPE" ]; then
            bad "$uncovered exists on this host, is not covered by any managed root, and carries"
            info "  '$(path_type "$uncovered")'. Re-run with --path '$uncovered' if it is workload data."
            failures=$((failures + 1))
        fi
    done

    report_avc
    if [ "${AVC_COUNT:-0}" -gt 0 ]; then
        bad "denials are still being recorded; do not transition to Enforcing yet."
        failures=$((failures + 1))
    elif [ "${AVC_COUNT:-0}" -lt 0 ]; then
        warn 'AVC state unknown (no ausearch); the Enforcing stage will refuse to run.'
        failures=$((failures + 1))
    fi

    head1 'Verdict'
    if [ "$failures" -eq 0 ]; then
        ok 'safe to proceed toward Enforcing'
        return 0
    fi
    bad "$failures check(s) failed; NOT safe to proceed toward Enforcing"
    return 1
}

# ----------------------------------------------------------------------------------- enforce

enforce() {
    head1 'Controlled Enforcing transition'
    if [ "$SELINUX_MODE" = 'Enforcing' ]; then
        ok 'already Enforcing; only the persisted config is checked'
    fi
    if [ "$APPLY" -ne 1 ]; then
        warn 'dry run: pass --apply to perform the transition'
    elif [ "$ASSUME_YES" -ne 1 ] && [ "$SELINUX_MODE" != 'Enforcing' ]; then
        printf '  Transition this host to Enforcing now? [y/N] '
        local answer; read -r answer
        case "$answer" in [yY]|[yY][eE][sS]) ;; *) warn 'aborted by operator'; return 1 ;; esac
    fi
    # setenforce first: it is instantly reversible with `setenforce 0` and needs no reboot. The
    # config file is only persisted after the runtime switch has been taken and re-verified, so a
    # host that misbehaves under Enforcing never reboots into it.
    run "$SUDO setenforce 1"
    if [ "$APPLY" -eq 1 ]; then
        say ''
        info 'Runtime mode is now Enforcing. Re-verifying before persisting the config...'
        AVC_SINCE="$(date '+%m/%d/%Y %H:%M:%S')"
        AVC_SINCE_EXPLICIT=1
        if ! verify; then
            bad 'post-transition verification failed. Reverting the runtime mode; config untouched.'
            $SUDO setenforce 0
            return 1
        fi
        run "$SUDO sed -i -E 's/^SELINUX=.*/SELINUX=enforcing/' /etc/selinux/config"
        ok 'Enforcing is now both active and persisted.'
        info "Revert at any time with: $SUDO setenforce 0"
        info "Revert persistently with: $SUDO sed -i -E 's/^SELINUX=.*/SELINUX=permissive/' /etc/selinux/config"
    fi
    return 0
}

# ---------------------------------------------------------------------------------- rollback

rollback() {
    head1 'Rollback'
    local file; file="$(state_file)"
    if [ ! -r "$file" ]; then
        warn "no recorded changes at $file; nothing to roll back"
        return 0
    fi
    [ "$APPLY" -eq 1 ] || warn 'dry run: pass --apply to perform the rollback'
    local path
    while read -r path; do
        [ -n "$path" ] || continue
        run "$SUDO semanage fcontext -d '${path}(/.*)?' || true"
        [ -e "$path" ] && run "$SUDO restorecon -RF '$path'" || true
    done < "$file"
    # Directories this script created and files it wrote are left in place: removing a node
    # storage root would destroy live PersistentVolume data.
    run "$SUDO rm -f '$file'"
    ok 'mappings this script added have been removed; storage contents were not touched.'
    return 0
}

# -------------------------------------------------------------------------------------- main

inspect_selinux
inspect_kubernetes
discover_pv_paths

case "$STAGE" in
    inspect)
        report_labels
        report_avc
        say ''
        info "Next: $0 validate"
        ;;
    validate)
        report_labels
        report_avc
        validate
        ;;
    normalize)
        validate || exit 1
        report_labels
        normalize
        say ''
        info "Next: $0 verify --since recent"
        ;;
    verify)
        verify
        ;;
    enforce)
        validate || exit 1
        verify || { bad 'refusing to transition to Enforcing while verification fails'; exit 1; }
        enforce
        ;;
    mark)
        head1 'Denial window'
        if [ "$APPLY" -eq 1 ]; then
            $SUDO install -d -m 0750 "$STATE_DIR"
            date '+%m/%d/%Y %H:%M:%S' | $SUDO tee "$(normalize_marker_file)" >/dev/null
            ok "verify will now count denials from $(cat "$(normalize_marker_file)")"
            info 'Labels and SELinux mode were not touched.'
        else
            warn 'dry run: pass --apply to move the window to now'
        fi
        ;;
    rollback)
        rollback
        ;;
    *)
        usage >&2; exit 2 ;;
esac