# The smoke tier, single-sourced in @norskvideo/ctl-dev-kit (conventions/smoke.yml)
# and drift-checked — the two per-repo lines are the ci-status-dispatch
# `product:` key and the `runner:` label array.
#
# One journey per product (04b s7.2): launch from the stored template, accept
# the source the manifest advertises, produce the observable it advertises —
# tests/smoke.spec.ts over @norskvideo/ctl-test-harness/smoke. The image under
# test is built HERE (`bun run build:image`), on the same runner that holds the
# licence, so the tier runs on pull requests and needs no cross-workflow state;
# the bytes build-image pushed are covered by ctl's manifest-driven job from the
# published tag instead. The daemon is the RELEASED norsk-ctl binary, as a
# customer would run it.
#
# Runner selection is a one-entry matrix carrying a JSON label array, because a
# string like '[ARM64, nvidia]' in runs-on is ONE label, not two. The runner
# MUST run Docker natively: `product add --image` publishes the control plane on
# a host port and the daemon polls 127.0.0.1:<port> for readiness, so on a
# docker-outside-of-docker pool (the x64 runners are containers) that loopback
# is the docker host's, never the runner's, and registration times out. Default
# '["ar-arm-vm"]' (native Docker, arm64 — the image is built for it here); the
# GPU box is '["ARM64","nvidia"]'.
name: smoke

on:
  pull_request:
  push:
    branches: [main]
  repository_dispatch:
    types: [ctl-candidate, pins-bumped]
  workflow_dispatch:
    inputs:
      ctl_channel:
        description: "norsk-ctl S3 channel to test against in place of the repo's pin (flake.nix ctlVersion). Empty runs the pin; release-gate dispatches 'rc' to report on a candidate (advisory: latest is not held on it)."
        type: string
        default: ""

permissions:
  contents: read

# Key the group on event_name so a push never queues behind an in-flight
# ctl-candidate validation. Nothing cancels a run in progress
# (no-cancel-in-progress.test.ts): a run queues, and GitHub keeps only the
# newest pending one. A cancelled validation would score as a red at
# release-collect.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: false

jobs:
  smoke:
    strategy:
      matrix:
        include:
          - runner: '["ar-arm-vm"]'
    runs-on: ${{ fromJSON(matrix.runner) }}
    steps:
      # Runs BEFORE checkout, and does two things: points the harness's temp base
      # out of the workspace for every later step in this job, and clears whatever
      # the old repo-local default left behind on this runner. The body carries the
      # why. It fails the job only when legacy litter survives -- which would take
      # the checkout down a step later anyway, but report nothing useful.
      - name: Route test-temp out of the workspace (pre-checkout)
        run: |
          set -uo pipefail
          # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
          # HOST daemon has not seen before is created by dockerd, as ROOT, on the
          # host -- even when the launch passes --container-user (reproduced
          # 2026-09-04). The harness's default base is repo-local for an OrbStack
          # inotify constraint that binds macOS dev and nothing here, so every run
          # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
          # cannot remove. That took commentary's checks AND integration red on
          # 2026-09-03. Use the sanctioned override, per-run and outside the
          # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
          base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$base"
          echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
          # Yesterday's bases hold dockerd-created root dirs, so a non-root rm
          # EACCESes. They cannot wedge a checkout, but they do fill the disk.
          for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
            docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
          done
          # LEGACY: a runner that ran the old default still carries a workspace
          # test-temp, and a checkout with clean:true dies on it. Reap the holders
          # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
          # RESTARTING leaked container re-creates the path as root seconds after
          # any clean. Match on mount SOURCE, not a label: the holders are untracked
          # by construction, because a killed run labels nothing.
          tt="$GITHUB_WORKSPACE/test-temp"
          [ -d "$tt" ] || exit 0
          stuck=""
          for c in $(docker ps -aq 2>/dev/null || true); do
            if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
              stuck="$stuck $c"
            fi
          done
          if [ -n "$stuck" ]; then
            echo "reaping containers holding mounts under $tt:$stuck"
            docker rm -f $stuck || true
          fi
          # The DIRECTORY, not just its contents: an empty but root-owned base
          # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
          # container unlink the leaf.
          docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
            sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
          # Fail HERE if it survived. The checkout fails on it either way, but
          # reports only an EACCES rmdir with no clue what was holding the path.
          if [ -e "$tt" ]; then
            echo "::error::$tt survived the pre-checkout clean"
            ls -lan "$tt" || true
            exit 1
          fi

      - uses: actions/checkout@v5
        with:
          clean: true

      # A containerised runner (docker-outside-of-docker) reaches host-published
      # ports via the host-gateway alias and can join norsk-net for direct reach;
      # a bare-metal host runner (the GPU box) must stay on localhost + publish.
      # Detect rather than hardcode, so one byte-identical file serves both.
      # NORSK_CTL_PRODUCT_REACH rides along because this step has already
      # established the fact the daemon would otherwise sniff from its own
      # cgroup: `product add --image` loopback-publishes the control plane and
      # polls it for readiness, and on a DooD runner that loopback is the docker
      # HOST's, so registration times out at 60s with nothing in the log saying
      # why. The daemon's own detection stays the default; here the probe that
      # already knows simply says so.
      - name: Probe runner docker topology and set NORSK_TEST_HOST + NORSK_TEST_NET
        run: |
          set -uo pipefail
          echo "hostname=$(hostname)"
          if docker inspect "$(hostname)" >/dev/null 2>&1; then
            echo "TOPO: hostname resolves to a container -> runner is containerized (DooD)"
            echo "NORSK_TEST_HOST=host.docker.internal" >> "$GITHUB_ENV"
            echo "NORSK_TEST_NET=direct" >> "$GITHUB_ENV"
            echo "NORSK_CTL_PRODUCT_REACH=container-address" >> "$GITHUB_ENV"
            echo "NORSK_RUNNER_OWNS_DOCKER=false" >> "$GITHUB_ENV"
          else
            echo "TOPO: hostname is not a container -> runner is host-based"
            echo "NORSK_TEST_HOST=localhost" >> "$GITHUB_ENV"
            echo "NORSK_TEST_NET=publish" >> "$GITHUB_ENV"
            echo "NORSK_CTL_PRODUCT_REACH=host" >> "$GITHUB_ENV"
            echo "NORSK_RUNNER_OWNS_DOCKER=true" >> "$GITHUB_ENV"
          fi

      # `product add --image` publishes the control plane on 127.0.0.1:14321. The
      # daemon means to reap it on shutdown (daemon.ts, model B) but only does so
      # gracefully, and this workflow cancels its own superseded runs -- a killed
      # daemon reaps nothing and the port stays held for every later job here.
      #
      # On 2026-08-30 something held it on ar-arm-vm from 11:18 to ~16:00 and
      # took out the next nine smoke and integration jobs across four repos.
      # What held it was never established: the job that last bound it PASSED
      # and shut down cleanly, and every add-path failure already removes its
      # own container -- so the holder may not have been a control plane at all.
      # "already allocated" is also what a dangling endpoint or a stranded
      # docker-proxy reports after an unclean removal.
      #
      # Hence: kill by PORT rather than by name, prune the endpoints a removal
      # can strand, and then PROVE the port binds instead of assuming it. This
      # branch only runs where the docker daemon is this runner's own and jobs
      # are serial, so there is nothing else here to protect. The GPU box runs
      # several runners as sibling CONTAINERS against one host daemon and is
      # excluded -- there, a blanket reap would kill a concurrent job.
      - name: Reap a stranded product control plane
        if: ${{ env.NORSK_RUNNER_OWNS_DOCKER == 'true' }}
        run: |
          set -uo pipefail
          port=14321
          bindable() { docker run --rm -p "127.0.0.1:$port:1" alpine true >/dev/null 2>&1; }

          held="$(docker ps -a --filter "publish=$port" --format '{{.Names}} {{.Status}} {{.Image}}')"
          if [ -n "$held" ]; then
            echo "::warning::$port was held on entry by: $held"
            docker rm -f $(docker ps -aq --filter "publish=$port") || true
          fi
          # A control plane that is stopped, or one whose publish docker no
          # longer reports, still gets removed: nothing here outlives a job.
          leftover="$(docker ps -aq --filter 'name=^norsk-product-')"
          [ -z "$leftover" ] || docker rm -f $leftover || true

          if ! bindable; then
            echo "::warning::$port still not bindable after removing containers -- pruning dangling endpoints"
            docker network prune -f || true
          fi
          if ! bindable; then
            echo "::error::cannot bind 127.0.0.1:$port; the journey would fail later and more obscurely"
            docker ps -a --filter "publish=$port" || true
            exit 1
          fi
          echo "$port is bindable"

      - name: Write the Norsk license (from the org secret)
        env:
          NORSK_LICENSE_V2: ${{ secrets.NORSK_LICENSE_V2 }}
        run: printf '%s' "$NORSK_LICENSE_V2" > "$RUNNER_TEMP/norsk-license.json"

      # Every norsk-ctl daemon starts the node MoQ relay (norsk-moq-relay) unless
      # told not to, and nothing removes it: it is a node singleton, and the reap
      # step only takes containers that mount this job's test-temp. So the first
      # test daemon on a shared runner left one, and every later one adopted it
      # (probe's smoke ran in relay mode on 2026-10-10; relay's smoke lost 4443
      # to it). This job's daemons run without the node relay and discovery, so
      # they neither start nor adopt one, and whatever is already on the host is
      # named here. It reports and never fails: the leftover may be someone's.
      - name: Isolate this job from node singletons other jobs left on this host
        run: |
          set -uo pipefail
          echo "NORSK_CTL_MOQ_RELAY_ENABLE=0" >> "$GITHUB_ENV"
          echo "NORSK_CTL_DISCOVERY_ENABLE=0" >> "$GITHUB_ENV"
          left="$(docker ps -a --filter label=norsk-ctl.role --format '{{.Names}} {{.Label "norsk-ctl.role"}} {{.Status}} (created {{.CreatedAt}})' 2>/dev/null || true)"
          if [ -z "$left" ]; then echo "no norsk-ctl containers on this host"; exit 0; fi
          echo "norsk-ctl containers already on this host (other jobs' running instances are expected):"
          echo "$left"
          while IFS= read -r line; do
            role="$(awk '{print $2}' <<<"$line")"
            case "$role" in
              moq-relay|discovery)
                echo "::warning title=node singleton left on this host::$line -- left by an earlier job (a daemon's auto-start, or a run killed before its reap). This job's daemons run without it and will not adopt it; remove it with docker rm -f if nothing on this host needs it." ;;
            esac
          done <<<"$left"

      # Pick the binary for THIS runner's architecture: the GPU box is aarch64,
      # the x64 pool is x86_64, and the wrong one dies with "Exec format error"
      # deep in the suite. No `|| true` on --version: a bad binary must fail here.
      #
      # WHICH binary: the version this commit pins, flake.nix ctlVersion -- the
      # daemon the dev shell launches, moved by bump.yml's nightly PR. It used
      # to be S3's moving `latest`, read at run time, so a result could change
      # with no product commit and a re-run was never a real repeat (funke went
      # red and green again on 2026-10-09 without a commit either time;
      # ci-workflows docs/dashboard/ci-fragility.md, proposal A). A ctl-candidate
      # dispatch still tests its payload's channel (rc) -- that is what it is
      # for -- and a by-hand ctl_channel may too. ctl-pinned-in-repo.test.ts.
      - name: Download the pinned norsk-ctl binary
        env:
          CTL_CHANNEL: ${{ github.event.client_payload.channel || inputs.ctl_channel }}
        run: |
          set -euo pipefail
          case "$(uname -m)" in
            x86_64) arch=linux-x64 ;;
            aarch64 | arm64) arch=linux-arm64 ;;
            *) echo "unsupported runner arch: $(uname -m)" >&2; exit 1 ;;
          esac
          S3="https://s3.eu-west-1.amazonaws.com/norsk.video/norsk-ctl"
          if [ -n "${CTL_CHANNEL:-}" ]; then
            ver="$(curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$CTL_CHANNEL")"
            echo "norsk-ctl $CTL_CHANNEL channel -> $ver ($arch), in place of the repo pin"
          else
            ver="$( { sed -nE 's/^[[:space:]]*ctlVersion = "([^"]+)";.*/\1/p' flake.nix 2>/dev/null || true; } | head -1)"
            [ -n "$ver" ] || { echo "::error::flake.nix pins no ctlVersion. CI runs the norsk-ctl this commit pins (the nightly bump.yml moves it); add ctlVersion = \"<version>\"; to flake.nix."; exit 1; }
            echo "norsk-ctl pinned by flake.nix ctlVersion -> $ver ($arch)"
          fi
          curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$ver/norsk-ctl-$ver-$arch" -o "$RUNNER_TEMP/norsk-ctl"
          chmod +x "$RUNNER_TEMP/norsk-ctl"
          "$RUNNER_TEMP/norsk-ctl" --version

      # Everything below runs dev-kit and test-harness code out of node_modules,
      # and `clean: true` on the checkout means this job starts without any. It
      # used to start with whatever an earlier run had left behind, which is how
      # the entitlement check below ran green for a day on a node_modules nobody
      # in the job had created, then died fleet-wide the first cold checkout.
      - name: Install dependencies
        run: nix develop .#build --command bun install --frozen-lockfile

      # Before anything is built or launched: does this licence actually name
      # this product? `product add` refuses one it does not, and that refusal
      # reaches the log as a 60-second readiness timeout indistinguishable from
      # a product that failed to start. Turnkeys make this routine — each ships
      # under a licence naming its own product, so a newly created one whose
      # licence was never minted should say so in seconds, by name.
      #
      # The product name is the repo's own PRODUCT_NAME, not the short board key
      # above (`funke` vs `funke-pegasus`): the licence names the product. A
      # Studio product keeps it in shared/src/version.ts, an sdk-app (norsk-relay)
      # in src/product/version.ts; the first that exists is read.
      - name: Check the licence entitles this product
        run: |
          set -euo pipefail
          vf=""
          for f in shared/src/version.ts src/product/version.ts; do [ -f "$f" ] && { vf="$f"; break; }; done
          [ -n "$vf" ] || { echo "::error::no shared/src/version.ts or src/product/version.ts"; exit 1; }
          product="$(sed -nE 's/.*PRODUCT_NAME[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' "$vf" | head -1)"
          [ -n "$product" ] || { echo "::error::could not read PRODUCT_NAME from $vf"; exit 1; }
          nix develop .#build --command bun \
            node_modules/@norskvideo/ctl-dev-kit/licence/check-entitlement.ts \
            "$RUNNER_TEMP/norsk-license.json" "$product"

      # Instance launch docker-composes the media + studio images; on a cold
      # runner the first pull overruns the per-test timeout. Pre-pull them (tags
      # from manifest.seed.json), plus whatever the repo lists in the optional
      # scripts/smoke/prepull-extra.sh hook (a driver image, a sidecar). An
      # sdk-app has no Studio pair and no seed: its hook pulls its engine.
      - name: Pre-pull the media + studio images
        run: |
          set -euo pipefail
          if [ -f manifest.seed.json ]; then
            for img in "$(jq -r '.latest.media' manifest.seed.json)" "$(jq -r '.latest.studio' manifest.seed.json)"; do
              echo "pre-pulling $img"
              docker pull "$img"
            done
          fi
          if [ -x scripts/smoke/prepull-extra.sh ]; then
            echo "running scripts/smoke/prepull-extra.sh"
            ./scripts/smoke/prepull-extra.sh
          fi

      # The image is built UNDER THE LICENSED REPO: a V2 licence entitles a
      # product by image repo (norskvideo/norsk-<x>-product), so `product add`
      # refuses a throwaway name. That repo is the PUBLISH_IMAGE line
      # build-image.yml already carries per repo; a build-only product (empty)
      # falls back to its build script default, which its own licence must name.
      - name: Build the image and run the smoke journey against it
        env:
          NORSK_CTL_BINARY: ${{ runner.temp }}/norsk-ctl
          NORSK_LICENSE_FILE: ${{ runner.temp }}/norsk-license.json
          # A direct-SDK product's (sdk-app's) licence identity: the sdk-client
          # cert + key it answers the engine's V2 handshake with, base64, as repo
          # secrets. Studio carries its own baked identity, so a Studio product
          # sets neither and these arrive empty.
          NORSK_SDK_CLIENT_CERT_B64: ${{ secrets.NORSK_SDK_CLIENT_CERT_B64 }}
          NORSK_SDK_CLIENT_KEY_B64: ${{ secrets.NORSK_SDK_CLIENT_KEY_B64 }}
          # Where the harness copies each instance's full engine logs
          # (/var/log/norsk -> debug.json, host-mounted under the store) before
          # teardown nukes them, plus — since ctl-test-harness 0.1.60 — each
          # instance's CONTAINER logs, which is the only copy of the studio
          # container's stdout. Uploaded below. `docker logs` only sees the
          # notice+ stdout subset and a decode-error flood can drown its tail;
          # this is the complete structured log at every level.
          #
          # Keyed on run id + attempt because these runners are SHARED between
          # repos: a bare $RUNNER_TEMP/norsk-logs is one directory nothing clears,
          # so a job uploaded whatever earlier jobs had left in it (observed: one
          # product's artifact carrying ten of another's instances). Same shape the
          # test-temp base uses, for the same reason.
          NORSK_CI_LOG_DIR: ${{ runner.temp }}/norsk-logs-${{ github.run_id }}-${{ github.run_attempt }}
        run: |
          # Outer pipefail so the step reports the journey's exit, not tee's --
          # otherwise a failed smoke run over the `| tee` below scores green.
          set -euo pipefail
          nix develop .#build --command bash -c '
            set -euo pipefail
            # A frozen install does not reliably relink a hoisted package whose
            # version moved: a stale test-harness once survived a bump and a new
            # export was "not found" at import. Drop the harness copy so the
            # pinned version runs. Belt and braces now the checkout is clean.
            rm -rf node_modules/@norskvideo/ctl-test-harness
            bun install --frozen-lockfile
            pub="$(sed -nE "s/^[[:space:]]*PUBLISH_IMAGE:[[:space:]]*\"?([^\" ]*)\"?[[:space:]]*$/\\1/p" .github/workflows/build-image.yml | head -1)"
            def="$(sed -nE "s/.*IMAGE_TAG:-([^}]+)\\}.*/\\1/p" deployment/build-image.sh | head -1)"
            export IMAGE_TAG="${pub:+$pub:smoke}"
            export IMAGE_TAG="${IMAGE_TAG:-$def}"
            echo "smoke image: $IMAGE_TAG"
            bun run build:image
            SMOKE_IMAGE_TAG="$IMAGE_TAG" bun run test:smoke
          ' 2>&1 | tee "$RUNNER_TEMP/smoke.log"

      # We pretty much always want the Norsk logs from a smoke run: the journey
      # runs the RELEASED media + studio images, and the harness dumps each
      # launched instance's container logs (diagnose hook) into the run output
      # at a failure -- but the containers are gone after teardown, so those
      # internals only survive if the run log is persisted. Upload it so the
      # engine's own log is downloadable, not buried in the step output.
      # always() so a red run keeps its evidence.
      - name: Upload smoke run log
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: smoke-log-attempt-${{ github.run_attempt }}
          path: ${{ runner.temp }}/smoke.log
          retention-days: 7

      # The FULL structured engine logs (debug.json et al.), copied off the
      # host store by the harness before teardown -- the actual place a
      # media-node death is recorded, which `docker logs` and its stdout flood
      # cannot show. if-no-files-found: ignore so a green run that wrote nothing
      # here (or a launch that never reached teardown) does not fail the step.
      - name: Upload Norsk engine logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: norsk-engine-logs-attempt-${{ github.run_attempt }}
          path: ${{ runner.temp }}/norsk-logs-${{ github.run_id }}-${{ github.run_attempt }}
          retention-days: 7
          if-no-files-found: ignore
      - name: Reap the containers this job launched
        # Whatever the tests did. A teardown cut short by a test timeout leaves
        # its instance running on the shared Docker daemon (see JOB_REAP_STEP in
        # conventions/workflow-shell.ts); everything this job launched mounts its
        # own test-temp, so that is what goes.
        if: always()
        run: |
          base="${NORSK_CTL_TEST_TMP:-}"
          tt="$GITHUB_WORKSPACE/test-temp"
          s=""
          for c in $(docker ps -aq); do
            src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
            if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi
          done
          if [ -n "$s" ]; then
            echo "::warning::removing containers this job left running:$s"
            docker rm -f $s
          fi

  # Report this pipeline's result to the aggregated product CI dashboard
  # (id3as/ci-workflows). !cancelled() so a real pass/fail reports, but a
  # cancelled/superseded run does NOT — a cancellation dispatched as failure
  # would false-red the board. Candidate validations report to ctl instead.
  notify:
    needs: smoke
    if: ${{ !cancelled() && github.event_name != 'repository_dispatch' }}
    runs-on: x64
    steps:
      # Runs BEFORE checkout, and does two things: points the harness's temp base
      # out of the workspace for every later step in this job, and clears whatever
      # the old repo-local default left behind on this runner. The body carries the
      # why. It fails the job only when legacy litter survives -- which would take
      # the checkout down a step later anyway, but report nothing useful.
      - name: Route test-temp out of the workspace (pre-checkout)
        run: |
          set -uo pipefail
          # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
          # HOST daemon has not seen before is created by dockerd, as ROOT, on the
          # host -- even when the launch passes --container-user (reproduced
          # 2026-09-04). The harness's default base is repo-local for an OrbStack
          # inotify constraint that binds macOS dev and nothing here, so every run
          # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
          # cannot remove. That took commentary's checks AND integration red on
          # 2026-09-03. Use the sanctioned override, per-run and outside the
          # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
          base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$base"
          echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
          # Yesterday's bases hold dockerd-created root dirs, so a non-root rm
          # EACCESes. They cannot wedge a checkout, but they do fill the disk.
          for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
            docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
          done
          # LEGACY: a runner that ran the old default still carries a workspace
          # test-temp, and a checkout with clean:true dies on it. Reap the holders
          # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
          # RESTARTING leaked container re-creates the path as root seconds after
          # any clean. Match on mount SOURCE, not a label: the holders are untracked
          # by construction, because a killed run labels nothing.
          tt="$GITHUB_WORKSPACE/test-temp"
          [ -d "$tt" ] || exit 0
          stuck=""
          for c in $(docker ps -aq 2>/dev/null || true); do
            if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
              stuck="$stuck $c"
            fi
          done
          if [ -n "$stuck" ]; then
            echo "reaping containers holding mounts under $tt:$stuck"
            docker rm -f $stuck || true
          fi
          # The DIRECTORY, not just its contents: an empty but root-owned base
          # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
          # container unlink the leaf.
          docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
            sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
          # Fail HERE if it survived. The checkout fails on it either way, but
          # reports only an EACCES rmdir with no clue what was holding the path.
          if [ -e "$tt" ]; then
            echo "::error::$tt survived the pre-checkout clean"
            ls -lan "$tt" || true
            exit 1
          fi

      - uses: actions/checkout@v5
        with:
          clean: true
      - id: meta
        run: |
          if [ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" = "true" ]; then
            echo "state=failure" >> "$GITHUB_OUTPUT"
          else
            echo "state=success" >> "$GITHUB_OUTPUT"
          fi
      - uses: ./.github/actions/ci-status-dispatch
        with:
          token: ${{ secrets.CI_DISPATCH_TOKEN }}
          product: __PRODUCT__
          pipeline: smoke
          status: ${{ steps.meta.outputs.state }}
      - name: Reap the containers this job launched
        # Whatever the tests did. A teardown cut short by a test timeout leaves
        # its instance running on the shared Docker daemon (see JOB_REAP_STEP in
        # conventions/workflow-shell.ts); everything this job launched mounts its
        # own test-temp, so that is what goes.
        if: always()
        run: |
          base="${NORSK_CTL_TEST_TMP:-}"
          tt="$GITHUB_WORKSPACE/test-temp"
          s=""
          for c in $(docker ps -aq); do
            src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
            if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi
          done
          if [ -n "$s" ]; then
            echo "::warning::removing containers this job left running:$s"
            docker rm -f $s
          fi

  # Report the rc-candidate validation result to the norsk-ctl release gate. Only
  # on a ctl-candidate dispatch; release-collect records it for the CI board.
  # Advisory: ctl promotes latest without waiting on it, and this repo takes a
  # new ctl only through its bump PR, which runs this tier on the new pin.
  report-candidate:
    needs: smoke
    if: ${{ always() && github.event_name == 'repository_dispatch' && github.event.action == 'ctl-candidate' }}
    runs-on: x64
    steps:
      - name: Fire ctl-candidate-result at norsk-ctl
        env:
          GH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }}
          # Lowercase on purpose: the drift gate masks every `product:` line, so
          # one per-repo key serves both the notify and the report jobs.
          product: __PRODUCT__
          VERSION: ${{ github.event.client_payload.version }}
          RESULT: ${{ needs.smoke.result }}
        run: |
          set -euo pipefail
          state=failure; [ "$RESULT" = "success" ] && state=success
          # `tier: smoke`: the release gate tallies per tier, so this green
          # cannot stand in for the integration tier's (docs/release-gate.md).
          body="$(jq -nc --arg p "$product" --arg v "$VERSION" --arg r "$state" \
            '{event_type:"ctl-candidate-result",client_payload:{product:$p,version:$v,result:$r,tier:"smoke"}}')"
          code="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
            -H "Authorization: Bearer $GH_TOKEN" \
            -H "Accept: application/vnd.github+json" \
            https://api.github.com/repos/id3as/norsk-ctl/dispatches -d "$body")"
          [ "$code" = "204" ] || { echo "::error::report to norsk-ctl failed (HTTP $code)"; exit 1; }