# Nightly wal-g base backup.
#
# wal-g backup-push needs direct PGDATA filesystem access, which only the
# db pod has. So instead of a separate backup container that talks to
# postgres over the network (the old pg_dump design), this CronJob runs a tiny
# kubectl image and `kubectl exec`s wal-g INSIDE the db pod — where wal-g, the
# WALG_*/AWS_* env, and PGDATA already live (see k8s/base/db/statefulset.yaml).
# No bespoke backup image, no sideload, no registry push.
#
# HA-safe: the command guards on pg_is_in_recovery()=f, so it only takes a base
# backup on the PRIMARY. On the standby (in recovery) it no-ops, and after a
# failover the new primary's CronJob transparently starts taking base backups —
# no per-cluster config.
#
# Result marker (src/lib/walg/backup-result.js; same contract as
# carbon/backup/compose-backup.sh): a run that exits 0 prints exactly one
# BACKUP_RESULT=pushed:<name> or BACKUP_RESULT=skipped:<reason> line, which
# `vibecarbon backup` and destroy read from the Job logs. The pushed line comes
# before retention, so a retention failure still names the base; it exits 3.
#
# One backup at a time per database: every Job (the schedule, and each
# `kubectl create job --from=cronjob/backup` the CLI starts, which
# concurrencyPolicy Forbid does not see) takes a lock inside the db container
# before wal-g runs. A held lock prints BACKUP_RESULT=skipped:already-running
# and the Job completes; LOCK_WAIT (seconds, set on the Job the CLI starts
# after a promotion) waits for it first. Two concurrent runs would each run
# `wal-g delete garbage`, which can delete the base the other is still pushing.
#
# Retries never re-push a landed base: restartPolicy Never (no in-place
# container restart) plus a podFailurePolicy that fails the Job at once on
# exit 3 (base pushed, retention failed). A failed push (exit 1, no marker)
# still gets a new pod under backoffLimit. podFailurePolicy is GA in
# Kubernetes 1.31, the line K3S_VERSION (src/lib/deploy/k8s/k3s.js) pins.
apiVersion: batch/v1
kind: CronJob
metadata:
  name: backup
  annotations:
    # Deploy-owned (see spec.schedule): Flux never garbage-collects the live one.
    kustomize.toolkit.fluxcd.io/prune: disabled
  labels:
    app: vibecarbon-backup
    component: backup
spec:
  # DEPLOY-owned, per environment: the CLI deploy and the deploy workflow fill
  # this in (scripts/render-backup-settings.mjs) from the environment backup
  # config. Flux does not build this object (see the vibecarbon-base patches
  # in k8s/flux/clusters/*), so it never sees the placeholder.
  schedule: "{{BACKUP_SCHEDULE}}"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      # BACKUP_JOB_ACTIVE_DEADLINE_S (src/backup.js) is this value: the CLI
      # waits for a Job as long as it may run.
      activeDeadlineSeconds: 3600
      backoffLimit: 2
      podFailurePolicy:
        rules:
          - action: FailJob
            onExitCodes:
              containerName: backup
              operator: In
              values: [3]
      template:
        metadata:
          labels:
            app: vibecarbon-backup
            component: backup
        spec:
          restartPolicy: Never
          serviceAccountName: backup-sa
          containers:
            - name: backup
              # Pinned public kubectl image — no build/sideload/push. Bump the
              # tag deliberately; `latest` would risk a silent client/server
              # skew on the scheduled run. Needs bash (the script below uses
              # `set -euo pipefail`) + kubectl on PATH, so the distroless
              # registry.k8s.io/kubectl is unusable. Was bitnami/kubectl:1.31
              # until Bitnami sunset their public Docker Hub catalog (2025) and
              # the tag vanished (`not found` → ImagePullBackOff); alpine/k8s
              # is the maintained drop-in (alpine base, bash + kubectl).
              image: "{{KUBECTL_IMAGE}}"
              imagePullPolicy: IfNotPresent
              env:
                - name: BACKUP_RETENTION_DAYS
                  valueFrom:
                    configMapKeyRef:
                      name: vibecarbon-backup-settings
                      key: BACKUP_RETENTION_DAYS
                      # Optional only so the script below can say what is wrong
                      # and name the fix; there is no default retention.
                      optional: true
                # The retirement marker (HA-correctness R7): a failover retired
                # this cluster, and the CLI patched VIBECARBON_ROLE=retired into
                # vibecarbon-config when it next reached it. Read per job, so a
                # patched ConfigMap reaches the next run with no restart.
                - name: VIBECARBON_ROLE
                  valueFrom:
                    configMapKeyRef:
                      name: vibecarbon-config
                      key: VIBECARBON_ROLE
                      optional: true
                # Where the TLS snapshot step below saves the vibecarbon-tls
                # Secret (s3://<backup bucket>/backups/<project>/<env>/tls/<ca>).
                # In the deploy-owned settings ConfigMap, never in
                # vibecarbon-config, which Flux resets from git. Unset: no
                # snapshot.
                - name: TLS_SNAPSHOT_S3_PREFIX
                  valueFrom:
                    configMapKeyRef:
                      name: vibecarbon-backup-settings
                      key: TLS_SNAPSHOT_S3_PREFIX
                      optional: true
              command:
                - /bin/bash
                - -c
                - |
                  set -euo pipefail
                  # A retired cluster (the old primary after a failover) holds a
                  # timeline the promoted cluster has left: a base backup plus
                  # delete before from here would land in, and prune, the
                  # promoted cluster backups. Refuse loudly, never a skip, before
                  # kubectl exec reaches wal-g. vibecarbon deploy rebuilds this
                  # cluster as the standby and clears the marker once it streams.
                  if [ "${VIBECARBON_ROLE:-}" = "retired" ]; then
                    echo "retired node: backups refused (run vibecarbon deploy to rebuild it as the standby)" >&2
                    exit 1
                  fi
                  RETAIN="${BACKUP_RETENTION_DAYS:-}"
                  # No default: the days come from the vibecarbon-backup-settings
                  # ConfigMap that a deploy creates per environment. It is spliced
                  # into the db-container script and its date arithmetic, so only a
                  # positive whole number of days is accepted.
                  case "$RETAIN" in
                    ""|*[!0-9]*|0|0[0-9]*)
                      echo "BACKUP_RETENTION_DAYS from ConfigMap vibecarbon-backup-settings is missing or not a positive whole number of days (got: $RETAIN). Run vibecarbon deploy for this environment (or the deploy workflow) to create it." >&2
                      exit 1
                      ;;
                  esac
                  # Seconds to wait for another backup run before skipping. It is
                  # spliced into the db-container command, so only a whole number.
                  LOCK_WAIT="${LOCK_WAIT:-0}"
                  case "$LOCK_WAIT" in
                    ""|*[!0-9]*)
                      echo "LOCK_WAIT must be a whole number of seconds (got: $LOCK_WAIT)" >&2
                      exit 1
                      ;;
                  esac
                  # Exec wal-g inside the db pod. The guard makes this a no-op on
                  # a standby (in recovery) so only the primary takes base backups.
                  # Exit 75 is the lock skip (marker printed): the Job completes.
                  rc=0
                  kubectl exec -n vibecarbon statefulset/db -c db -- env "LOCK_WAIT=$LOCK_WAIT" bash -c '
                    set -euo pipefail
                    # ONE BACKUP AT A TIME per database, whichever Job runs this:
                    # the lock is a file in the db container, held for the whole
                    # run. busybox flock has no -w, so the bounded wait polls -n.
                    # VC_BACKUP_LOCKFILE exists for the script tests only.
                    # /tmp is writable by the postgres user and this opens as root:
                    # a symlink at the path is refused, and >> never truncates.
                    lock_path="${VC_BACKUP_LOCKFILE:-/tmp/vc-backup.lock}"
                    if [ -L "$lock_path" ]; then
                      echo "backup lock $lock_path is a symlink, refusing" >&2
                      exit 1
                    fi
                    exec 9>>"$lock_path"
                    lock_until=$(( $(date +%s) + LOCK_WAIT ))
                    until flock -n 9; do
                      if [ "$(date +%s)" -ge "$lock_until" ]; then
                        echo "Backup already running, skipping" >&2
                        echo "BACKUP_RESULT=skipped:already-running"
                        exit 75
                      fi
                      sleep 2
                    done
                    # wal-g backup-push connects to Postgres (pg_backup_start) using
                    # libpq env. Unset, PGUSER defaults to the container OS user
                    # "root" → "role \"root\" does not exist". And only the cluster
                    # superuser may call pg_backup_start, and on our postgres:17
                    # image that is the postgres role. So pin the libpq
                    # connection explicitly.
                    # (wal-push/archiving and backup-fetch/restore do not connect to
                    # PG, which is why this only bit backup-push. RCA 2026-05-30.)
                    export PGUSER=postgres PGHOST=localhost PGPORT=5432 PGDATABASE=postgres
                    # This database lineage folder: walg-env.sh (mounted from the
                    # vibecarbon-wal-archive ConfigMap) turns WALG_S3_PREFIX_BASE into
                    # the folder named by the system identifier in PGDATA, so the push,
                    # the fence read and the retention act on this lineage only. One it
                    # cannot read exits 1 with no marker. WALG_ENV_SH exists for the
                    # script tests only. A segment name already archived with other
                    # bytes is refused, never overwritten (as on the db container).
                    . "${WALG_ENV_SH:-/etc/postgresql/walg-env.sh}" || exit 1
                    export WALG_PREVENT_WAL_OVERWRITE=true
                    # WRITE-GUARD (src/lib/deploy/walg-role.js): only a node whose wal-g role
                    # file says primary base-backs-up into the lineage prefix (walg-env.sh reads
                    # it into VC_WALG_ROLE). This closes the bring-up window the
                    # pg_is_in_recovery() check below cannot (an independent primary not yet in
                    # recovery). No role, or an unknown one, refuses with no marker.
                    case "${VC_WALG_ROLE:-}" in
                      primary) ;;
                      standby)
                        echo "wal-g role standby, skipping base backup (only the primary writes the lineage prefix)."
                        echo "BACKUP_RESULT=skipped:standby-role"
                        exit 0
                        ;;
                      *)
                        echo "backup refused: no wal-g role in ${VC_WALG_ROLE_FILE:-/var/lib/vibecarbon/role/walg-role} (vibecarbon deploy writes it)." >&2
                        exit 1
                        ;;
                    esac
                    # The recovery probe must FAIL LOUD when it cannot prove
                    # primary vs standby. The previous form compared $(psql …)
                    # inside a test: a failed psql (database starting up, exec
                    # transport broken) collapsed to "", read as standby, and
                    # the job exited 0 having backed up NOTHING (latent
                    # false-green found in the 2026-08-16 backup RCA). Only a
                    # literal t skips; only a literal f proceeds.
                    REC="$(psql -U postgres -d postgres -tAc "SELECT pg_is_in_recovery()")" || {
                      echo "recovery probe failed (psql exit $?) — refusing to guess primary/standby."
                      exit 1
                    }
                    case "$REC" in
                      t)
                        echo "db is in recovery (standby), skipping base backup."
                        echo "BACKUP_RESULT=skipped:in-recovery"
                        exit 0
                        ;;
                      f) ;;
                      *)
                        # No quotes around $REC: a literal single quote here
                        # would end the bash -c single-quoted wrapper above.
                        echo "unexpected pg_is_in_recovery() output: $REC — refusing to guess."
                        exit 1
                        ;;
                    esac
                    # BACKUP-PREFIX FENCE (src/lib/deploy/backup-fence.js): a failover wrote
                    # vibecarbon-fence.json into this prefix, naming the promoted primary database
                    # and the timeline it promoted onto. An old primary that rebooted unreached is
                    # the same database on an OLDER timeline: its base backup would become LATEST
                    # and its retention would prune the promoted node backups. Refuse loudly.
                    # No fence object (an env that never failed over) is allowed; one that cannot
                    # be read refuses, since nothing then proves this node is the primary.
                    fence="$(wal-g st cat vibecarbon-fence.json 2>&1)" || {
                      case "$fence" in
                        *"not found in storage"*) fence="" ;;
                        *)
                          echo "backup refused: could not read the backup-prefix fence (vibecarbon-fence.json): $fence" >&2
                          exit 1
                          ;;
                      esac
                    }
                    if [ -n "$fence" ]; then
                      fence_tl="$(printf %s "$fence" | grep -o "\"timeline\":[0-9]*" | grep -o "[0-9]*$")" || true
                      fence_sysid="$(printf %s "$fence" | grep -o "\"systemIdentifier\":\"[0-9]*\"" | grep -o "[0-9][0-9]*")" || true
                      local_sysid="$(psql -U postgres -d postgres -tAc "SELECT system_identifier FROM pg_control_system()")" || true
                      local_tl="$(psql -U postgres -d postgres -tAc "SELECT substr(pg_walfile_name(pg_current_wal_lsn()), 1, 8)")" || true
                      if ! [[ "$fence_tl" =~ ^[0-9]+$ && "$local_sysid" =~ ^[0-9]+$ && "$local_tl" =~ ^[0-9A-Fa-f]{8}$ ]]; then
                        echo "backup refused: could not compare this node with the backup-prefix fence (timeline $fence_tl, local $local_tl)" >&2
                        exit 1
                      fi
                      if [ "$local_sysid" = "$fence_sysid" ] && [ "$((16#$local_tl))" -lt "$fence_tl" ]; then
                        echo "backup prefix fenced to another primary (timeline $fence_tl): this node is on timeline $((16#$local_tl)), a primary a failover retired; backups refused (run vibecarbon status, then vibecarbon deploy to rebuild it as the standby)" >&2
                        exit 1
                      fi
                    fi
                    # RETENTION, BEFORE the push. It reads every sentinel in the
                    # lineage, and on Hetzner Object Storage a sentinel written
                    # seconds ago can answer 404: a backup on 2026-11-06 pushed its
                    # base, then failed reading the sentinel of that same base.
                    # Retention is by time and never needs it, so running first
                    # takes that read away rather than retrying it, and it can never
                    # touch the base about to land. A failure does not stop the
                    # push: it is reported after it as exit 3, which the
                    # podFailurePolicy turns into a failed Job (no new pod, so no
                    # second push). An empty lineage is fine: on wal-g 3.0.9 both
                    # commands exit 0 with nothing to do. Sweep sentinel-less
                    # orphans (interrupted pushes) before the delete, since
                    # it hard-crashes on a backup whose
                    # stop-sentinel never got written, and retries cannot heal a
                    # permanently missing object (2026-08-30 RCA). Safe under the
                    # db-container lock above, which every backup Job takes, so no
                    # concurrent push can look like an orphan mid-flight.
                    # Retention is by TIME and keeps the full retention window. The
                    # cutoff is that many days ago; the target is the NEWEST base that
                    # finished at or before it, and delete before FIND_FULL on that
                    # base removes what precedes it, so the base and its WAL stay and
                    # a restore to any moment in the window has its base. No base that
                    # old: nothing is deleted, and the newest base always survives.
                    # The list is parsed without jq (Alpine busybox db image): the
                    # cutoff is epoch arithmetic, the JSON is split with tr and sed.
                    # It acts on this lineage folder only (walg-env.sh). The fence
                    # object is read back after the delete, and one that changed or
                    # vanished fails retention.
                    # NO APOSTROPHES in this wrapper, comments included: a
                    # single quote ends the bash -c string (enforced by
                    # tests/unit/template/embedded-shell-syntax.test.ts;
                    # run 33319907805 is the day one broke every backup).
                    RETAIN_DAYS='"${RETAIN}"'
                    retain_rc=0
                    retain_out="$( { cutoff="$(date -u -d "@$(( $(date +%s) - RETAIN_DAYS * 86400 ))" +%Y-%m-%dT%H:%M:%SZ)" && [ -n "$cutoff" ] && wal-g delete garbage BACKUPS --confirm && list="$(wal-g backup-list --detail --json)" && target="$(printf "%s" "$list" | tr "}" "\n" | sed -n "s/.*\"backup_name\":\"\([^\"]*\)\".*\"finish_time\":\"\([^\"]*\)\".*/\2 \1/p" | awk -v c="$cutoff" "substr(\$1,1,19) <= substr(c,1,19)" | sort | tail -n 1 | cut -d " " -f 2)" && { if [ -n "$target" ]; then wal-g delete before FIND_FULL "$target" --confirm; else echo "no base backup is older than ${RETAIN_DAYS} days, nothing to delete"; fi; } && { [ -z "$fence" ] || [ "$(wal-g st cat vibecarbon-fence.json 2>&1)" = "$fence" ] || { echo "the backup-prefix fence (vibecarbon-fence.json) changed or vanished during retention"; false; }; }; } 2>&1)" || retain_rc=$?
                    printf "%s\n" "$retain_out" >&2
                    # The push. wal-g logs to stderr only, and names the base on a
                    # Started line and, once the sentinel is written, on a Wrote
                    # backup with name line. Only the Wrote line proves a pushed
                    # base, so the marker takes its name from it; a push that
                    # exits 0 without it fails rather than guess.
                    push_started="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
                    push_out="$(wal-g backup-push "$PGDATA" 2>&1)" || {
                      printf "%s\n" "$push_out" >&2
                      exit 1
                    }
                    printf "%s\n" "$push_out" >&2
                    name="$(printf "%s\n" "$push_out" | grep -o "Wrote backup with name base_[0-9A-Za-z_]*" | tail -n 1 | sed "s/^Wrote backup with name //")" || true
                    if [ -z "$name" ]; then
                      echo "backup-push exited 0 but printed no backup name (no Wrote backup with name line), so nothing proves what landed" >&2
                      exit 1
                    fi
                    echo "BACKUP_RESULT=pushed:$name"
                    # WAL GAP RECORD (wal-archive.sh drops a segment only under disk pressure
                    # and records it here): a drop before this base started is behind it now,
                    # so its entry goes; one recorded while the push ran stays (this base needs
                    # that WAL). The record moves aside first, so an entry wal-archive.sh
                    # appends meanwhile lands in a new file and is never lost. It runs as the
                    # postgres user (gosu): this shell is root, and PGDATA is the postgres
                    # user directory, so a root write there could follow a planted symlink. A
                    # symlinked record is refused. A failure here only warns: the base is in
                    # the bucket.
                    gap="$PGDATA/vibecarbon-wal-gap"
                    if [ -e "$gap" ] || [ -e "$gap.clearing" ] || [ -L "$gap" ] || [ -L "$gap.clearing" ]; then
                      gosu postgres sh -c "
                        g=\"\$1\"; s=\"\$2\"
                        for f in \"\$g\" \"\$g.clearing\"; do
                          if [ -L \"\$f\" ]; then echo \"refusing the WAL gap record \$f: it is a symlink\" >&2; exit 1; fi
                        done
                        { [ ! -f \"\$g\" ] || mv -f \"\$g\" \"\$g.clearing\"; } &&
                          kept=\"\$(awk -v s=\"\$s\" \"\\\$2 >= s\" \"\$g.clearing\")\" &&
                          { [ -z \"\$kept\" ] || printf \"%s\\n\" \"\$kept\" >> \"\$g\"; } &&
                          rm -f \"\$g.clearing\"
                      " sh "$gap" "$push_started" || echo "could not clear the WAL gap record $gap after pushing $name" >&2
                    fi
                    if [ "$retain_rc" -ne 0 ]; then
                      echo "retention failed before pushing $name: $(printf "%s\n" "$retain_out" | grep -v "^$" | tail -n 1)" >&2
                      exit 3
                    fi
                  ' || rc=$?
                  # TLS SNAPSHOT: after a run that reached the push (0, its
                  # skips included, 3, base pushed with retention failed, and
                  # 75, another Job holds the lock), save the vibecarbon-tls
                  # Secret beside the base, never after a refusal or a failed
                  # push (1). The Secret streams from kubectl straight into
                  # wal-g st put in the db container (which holds the bucket
                  # credentials): never printed, never a file on this pod.
                  # Only a Secret an ACME issuer signed is saved; the standby
                  # self-signed one would replace a good snapshot. Prints to
                  # stderr only and never changes rc. No jq: the annotation
                  # is read with jsonpath.
                  if [ "$rc" -eq 0 ] || [ "$rc" -eq 3 ] || [ "$rc" -eq 75 ]; then
                    tls_prefix="${TLS_SNAPSHOT_S3_PREFIX:-}"
                    if ! [[ "$tls_prefix" =~ ^s3://[A-Za-z0-9.-]+/[A-Za-z0-9._/-]+$ ]]; then
                      echo "tls snapshot skipped: no valid TLS_SNAPSHOT_S3_PREFIX in ConfigMap vibecarbon-backup-settings" >&2
                    else
                      tls_issuer="$(timeout 60 kubectl get secret vibecarbon-tls -n vibecarbon -o jsonpath="{.metadata.annotations.cert-manager\.io/issuer-name}" 2>/dev/null)" || tls_issuer=""
                      # The prefix ends in its CA folder (tls/prod or
                      # tls/staging); the issuer CA is staging when its name
                      # says so (caSlug). A snapshot under the other folder
                      # would be restored as the wrong CA.
                      tls_ca="${tls_prefix##*/}"
                      tls_issuer_ca=prod
                      if [[ "$tls_issuer" == *staging* ]]; then tls_issuer_ca=staging; fi
                      if [ -z "$tls_issuer" ] || [ "$tls_issuer" = "vibecarbon-standby-selfsigned" ]; then
                        echo "tls snapshot skipped: Secret vibecarbon-tls is missing or not issued by an ACME issuer (issuer: ${tls_issuer:-none})" >&2
                      elif [ "$tls_issuer_ca" != "$tls_ca" ]; then
                        echo "tls snapshot skipped: Secret vibecarbon-tls was issued by ${tls_issuer}, another CA than the ${tls_ca} folder of the snapshot prefix" >&2
                      # Bounded (busybox timeout), so a hung exec cannot run
                      # the Job into its deadline. The upload bound is the
                      # shorter one: if the read hangs, the upload is killed
                      # first and never completes an object from a short stream.
                      elif timeout 130 kubectl get secret vibecarbon-tls -n vibecarbon -o json |
                        timeout 120 kubectl exec -i -n vibecarbon statefulset/db -c db -- env WALG_S3_PREFIX="$TLS_SNAPSHOT_S3_PREFIX" wal-g st put -s -f --no-compress --no-encrypt k8s-secret.json >&2; then
                        echo "tls snapshot saved to ${tls_prefix}/k8s-secret.json" >&2
                      else
                        tls_rc=$?
                        if [ "$tls_rc" -eq 124 ] || [ "$tls_rc" -eq 143 ]; then
                          echo "tls snapshot failed: ${tls_prefix}/k8s-secret.json timed out after 120s" >&2
                        else
                          echo "tls snapshot failed: ${tls_prefix}/k8s-secret.json was not written (see the error above)" >&2
                        fi
                      fi
                    fi
                  fi
                  if [ "$rc" -eq 75 ]; then
                    exit 0
                  fi
                  exit "$rc"
              resources:
                requests:
                  cpu: 50m
                  memory: 64Mi
                limits:
                  cpu: 250m
                  memory: 128Mi
              securityContext:
                allowPrivilegeEscalation: false
          securityContext:
            seccompProfile:
              type: RuntimeDefault
