# Build + publish a PRODUCT container image, multi-arch (linux/amd64 +
# linux/arm64), to Docker Hub. Single-sourced in @norskvideo/ctl-dev-kit
# (conventions/build-image.yml) and drift-gated -- edit the dev-kit source, never
# the copy.
#
# Multi-arch house style: native build per arch (no QEMU), each arch pushes its
# own `:<label>-<arch>` tag to Docker Hub, then a publish job combines them with
# `docker buildx imagetools create` into the multi-arch manifest. Each publish
# writes two tags:
#   <repo>:<label>    immutable, label = YYYY-MM-DD-<sha8> (what deployments pin)
#   <repo>:<channel>  moving (latest | rc | nightly)
#
# The one per-repo value is the masked PUBLISH_IMAGE env: set to a product's
# Docker Hub repo (norskvideo org) it builds + publishes both arches; left empty
# the build runs as a standalone-build proof (both arches) and publishes nothing
# -- for a product that is not deployed yet.
#
# CHANNEL: push-to-main publishes `latest` (matches the retired in-monorepo
# pipeline, so :latest + the dated tags stay fresh for installers/deployments).
# workflow_dispatch can publish any channel. When the product channel model
# (rc -> latest promotion) lands, flip the push default to `nightly` and let the
# promotion gate move `latest`.
#
# BOT PRs: on a pull request from a bot branch (bump/nightly, and the retired
# bots' upgrade-latest/*, sync-dev-kit/* and chore/sync-ctl-packages, which the
# PR that brings bump.yml rides on) the workflow builds both arches and
# runs the guides gate exactly as a push would (the deterministic tiers gate;
# the engine tier reports beside it), and publishes, records and
# reports nothing. A bump reaches main only once that has passed on it, and
# bot-prs merges on it (ci-workflows docs/dashboard/ci-fragility.md, proposal
# B): a dev-kit sync that brought a newer copy of THIS file broke
# oracle-reasoning's build on main on 2026-10-09, after a green unit gate
# landed it. A human PR does not pay for the build; gate-checks is skipped on a
# PR because the PR's own checks run already covers it.
#
# Repo-local for now; to be hoisted into id3as/ci-workflows as a reusable
# workflow (RFC 0001 Workstream E).
name: build-image

on:
  push:
    branches: [main]
  pull_request:
  workflow_dispatch:
    inputs:
      channel:
        description: "Moving channel tag to publish (latest | rc | nightly)."
        type: string
        default: latest
      dry_run:
        description: "Build + stage both arches, skip the Docker Hub publish."
        type: boolean
        default: false

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

env:
  # The Docker Hub repo this product publishes to (norskvideo org, e.g.
  # norskvideo/norsk-<product>-product). Empty string = build-only proof, no
  # publish (a product not yet deployed). This is the one per-repo env the drift
  # gate masks; set it in the product repo, never in the dev-kit canonical.
  PUBLISH_IMAGE: ""

jobs:
  # Compute the immutable label ONCE, so both arch builds and the manifest share
  # it (a per-job `date` could straddle midnight and disagree).
  label:
    if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && (github.head_ref == 'bump/nightly' || startsWith(github.head_ref, 'upgrade-latest/') || startsWith(github.head_ref, 'sync-dev-kit/') || github.head_ref == 'chore/sync-ctl-packages')) }}
    runs-on: x64
    outputs:
      label: ${{ steps.gen.outputs.label }}
      channel: ${{ steps.gen.outputs.channel }}
      # `publish` is derived here (in a step, where the env context is readable)
      # because the env context is NOT available in a job-level `if:` -- the
      # publish job gates on this output instead of on PUBLISH_IMAGE directly.
      publish: ${{ steps.gen.outputs.publish }}
    steps:
      # Runs BEFORE checkout, and does two things: points the harness's temp base
      # out of the workspace for every later step in this job, and clears whatever
      # the old repo-local default left behind on this runner. The body carries the
      # why. It fails the job only when legacy litter survives -- which would take
      # the checkout down a step later anyway, but report nothing useful.
      - name: Route test-temp out of the workspace (pre-checkout)
        run: |
          set -uo pipefail
          # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
          # HOST daemon has not seen before is created by dockerd, as ROOT, on the
          # host -- even when the launch passes --container-user (reproduced
          # 2026-09-04). The harness's default base is repo-local for an OrbStack
          # inotify constraint that binds macOS dev and nothing here, so every run
          # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
          # cannot remove. That took commentary's checks AND integration red on
          # 2026-09-03. Use the sanctioned override, per-run and outside the
          # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
          base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$base"
          echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
          # Yesterday's bases hold dockerd-created root dirs, so a non-root rm
          # EACCESes. They cannot wedge a checkout, but they do fill the disk.
          for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
            docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
          done
          # LEGACY: a runner that ran the old default still carries a workspace
          # test-temp, and a checkout with clean:true dies on it. Reap the holders
          # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
          # RESTARTING leaked container re-creates the path as root seconds after
          # any clean. Match on mount SOURCE, not a label: the holders are untracked
          # by construction, because a killed run labels nothing.
          tt="$GITHUB_WORKSPACE/test-temp"
          [ -d "$tt" ] || exit 0
          stuck=""
          for c in $(docker ps -aq 2>/dev/null || true); do
            if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
              stuck="$stuck $c"
            fi
          done
          if [ -n "$stuck" ]; then
            echo "reaping containers holding mounts under $tt:$stuck"
            docker rm -f $stuck || true
          fi
          # The DIRECTORY, not just its contents: an empty but root-owned base
          # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
          # container unlink the leaf.
          docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
            sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
          # Fail HERE if it survived. The checkout fails on it either way, but
          # reports only an EACCES rmdir with no clue what was holding the path.
          if [ -e "$tt" ]; then
            echo "::error::$tt survived the pre-checkout clean"
            ls -lan "$tt" || true
            exit 1
          fi

      - uses: actions/checkout@v5
        with:
          clean: true
      - id: gen
        run: |
          label="$(date -u +%Y-%m-%d)-$(git rev-parse --short=8 HEAD)"
          channel="${{ github.event.inputs.channel || 'latest' }}"
          if [ -n "${PUBLISH_IMAGE}" ]; then publish=true; else publish=false; fi
          echo "label=${label}" >> "$GITHUB_OUTPUT"
          echo "channel=${channel}" >> "$GITHUB_OUTPUT"
          echo "publish=${publish}" >> "$GITHUB_OUTPUT"
          echo "label=${label} channel=${channel} publish=${publish}"
      - name: Reap the containers this job launched
        # Whatever the tests did. A teardown cut short by a test timeout leaves
        # its instance running on the shared Docker daemon (see JOB_REAP_STEP in
        # conventions/workflow-shell.ts); everything this job launched mounts its
        # own test-temp, so that is what goes.
        if: always()
        run: |
          base="${NORSK_CTL_TEST_TMP:-}"
          tt="$GITHUB_WORKSPACE/test-temp"
          s=""
          for c in $(docker ps -aq); do
            src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
            if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi
          done
          if [ -n "$s" ]; then
            echo "::warning::removing containers this job left running:$s"
            docker rm -f $s
          fi

  build:
    needs: label
    strategy:
      fail-fast: false
      matrix:
        include:
          - runner: x64
            arch: amd64
          - runner: ar-arm-vm
            arch: arm64
    runs-on: ${{ matrix.runner }}
    steps:
      # Runs BEFORE checkout, and does two things: points the harness's temp base
      # out of the workspace for every later step in this job, and clears whatever
      # the old repo-local default left behind on this runner. The body carries the
      # why. It fails the job only when legacy litter survives -- which would take
      # the checkout down a step later anyway, but report nothing useful.
      - name: Route test-temp out of the workspace (pre-checkout)
        run: |
          set -uo pipefail
          # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
          # HOST daemon has not seen before is created by dockerd, as ROOT, on the
          # host -- even when the launch passes --container-user (reproduced
          # 2026-09-04). The harness's default base is repo-local for an OrbStack
          # inotify constraint that binds macOS dev and nothing here, so every run
          # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
          # cannot remove. That took commentary's checks AND integration red on
          # 2026-09-03. Use the sanctioned override, per-run and outside the
          # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
          base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$base"
          echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
          # Yesterday's bases hold dockerd-created root dirs, so a non-root rm
          # EACCESes. They cannot wedge a checkout, but they do fill the disk.
          for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
            docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
          done
          # LEGACY: a runner that ran the old default still carries a workspace
          # test-temp, and a checkout with clean:true dies on it. Reap the holders
          # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
          # RESTARTING leaked container re-creates the path as root seconds after
          # any clean. Match on mount SOURCE, not a label: the holders are untracked
          # by construction, because a killed run labels nothing.
          tt="$GITHUB_WORKSPACE/test-temp"
          [ -d "$tt" ] || exit 0
          stuck=""
          for c in $(docker ps -aq 2>/dev/null || true); do
            if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
              stuck="$stuck $c"
            fi
          done
          if [ -n "$stuck" ]; then
            echo "reaping containers holding mounts under $tt:$stuck"
            docker rm -f $stuck || true
          fi
          # The DIRECTORY, not just its contents: an empty but root-owned base
          # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
          # container unlink the leaf.
          docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
            sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
          # Fail HERE if it survived. The checkout fails on it either way, but
          # reports only an EACCES rmdir with no clue what was holding the path.
          if [ -e "$tt" ]; then
            echo "::error::$tt survived the pre-checkout clean"
            ls -lan "$tt" || true
            exit 1
          fi

      - uses: actions/checkout@v5
        with:
          clean: true

      - name: Build product image (copy-only, via the dev-kit driver, native ${{ matrix.arch }})
        run: |
          if [ -n "${PUBLISH_IMAGE}" ]; then
            IMAGE_TAG="${PUBLISH_IMAGE}:${{ needs.label.outputs.label }}-${{ matrix.arch }}"
          else
            IMAGE_TAG="norsk-ctl-product:${{ needs.label.outputs.label }}-${{ matrix.arch }}"
          fi
          echo "IMAGE_TAG=${IMAGE_TAG}" >> "$GITHUB_ENV"
          export IMAGE_TAG
          nix develop .#build --command bash -c '
            set -euo pipefail
            bun install --frozen-lockfile
            bun run build:image
          '

      - name: Log in to Docker Hub
        if: ${{ env.PUBLISH_IMAGE != '' && github.event_name != 'pull_request' }}
        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
        with:
          username: norskvideo
          password: ${{ secrets.DOCKER_PAT }}

      - name: Push the per-arch image to Docker Hub
        if: ${{ env.PUBLISH_IMAGE != '' && github.event_name != 'pull_request' }}
        run: docker push "$IMAGE_TAG"
      - name: Reap the containers this job launched
        # Whatever the tests did. A teardown cut short by a test timeout leaves
        # its instance running on the shared Docker daemon (see JOB_REAP_STEP in
        # conventions/workflow-shell.ts); everything this job launched mounts its
        # own test-temp, so that is what goes.
        if: always()
        run: |
          base="${NORSK_CTL_TEST_TMP:-}"
          tt="$GITHUB_WORKSPACE/test-temp"
          s=""
          for c in $(docker ps -aq); do
            src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
            if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi
          done
          if [ -n "$s" ]; then
            echo "::warning::removing containers this job left running:$s"
            docker rm -f $s
          fi

  # THE PUBLISH GATE. Nothing is published -- no dated tag, no channel tag, no
  # channels row -- until both halves pass: the repo's own checks, and its guides
  # (the end-to-end journey the nightly manual is generated from). Until
  # 2026-10-06 publish needed only [label, build], so a bot push that passed unit
  # tests shipped a probe whose preview was broken for four days, while the one
  # test that knew sat red in a nightly nobody saw (task 57).
  gate-checks:
    if: ${{ github.event_name != 'pull_request' }}
    uses: ./.github/workflows/checks.yml
    with:
      report: false
    secrets: inherit

  # The guides run from the SAME commit as the image, exactly as the nightly
  # publish-docs.yml runs them -- these steps are its steps (a convention test
  # holds them identical). Skipped, with the reason carried to the published row
  # and the board, for a build-only repo, a product with no guides, or one that
  # opted out in .github/publish-gate.json.
  gate-guides:
    needs: label
    runs-on: x64
    # No doc-guides concurrency group: the deterministic tiers launch no
    # instances, so there is nothing to serialise, and a pending job in a group
    # is cancelled by the next one queued. guides-engine takes the group.
    outputs:
      guides: ${{ steps.plan.outputs.guides }}
      reason: ${{ steps.plan.outputs.reason }}
      engine: ${{ steps.plan.outputs.engine }}
    steps:
      # Runs BEFORE checkout, and does two things: points the harness's temp base
      # out of the workspace for every later step in this job, and clears whatever
      # the old repo-local default left behind on this runner. The body carries the
      # why. It fails the job only when legacy litter survives -- which would take
      # the checkout down a step later anyway, but report nothing useful.
      - name: Route test-temp out of the workspace (pre-checkout)
        run: |
          set -uo pipefail
          # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
          # HOST daemon has not seen before is created by dockerd, as ROOT, on the
          # host -- even when the launch passes --container-user (reproduced
          # 2026-09-04). The harness's default base is repo-local for an OrbStack
          # inotify constraint that binds macOS dev and nothing here, so every run
          # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
          # cannot remove. That took commentary's checks AND integration red on
          # 2026-09-03. Use the sanctioned override, per-run and outside the
          # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
          base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$base"
          echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
          # Yesterday's bases hold dockerd-created root dirs, so a non-root rm
          # EACCESes. They cannot wedge a checkout, but they do fill the disk.
          for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
            docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
          done
          # LEGACY: a runner that ran the old default still carries a workspace
          # test-temp, and a checkout with clean:true dies on it. Reap the holders
          # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
          # RESTARTING leaked container re-creates the path as root seconds after
          # any clean. Match on mount SOURCE, not a label: the holders are untracked
          # by construction, because a killed run labels nothing.
          tt="$GITHUB_WORKSPACE/test-temp"
          [ -d "$tt" ] || exit 0
          stuck=""
          for c in $(docker ps -aq 2>/dev/null || true); do
            if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
              stuck="$stuck $c"
            fi
          done
          if [ -n "$stuck" ]; then
            echo "reaping containers holding mounts under $tt:$stuck"
            docker rm -f $stuck || true
          fi
          # The DIRECTORY, not just its contents: an empty but root-owned base
          # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
          # container unlink the leaf.
          docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
            sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
          # Fail HERE if it survived. The checkout fails on it either way, but
          # reports only an EACCES rmdir with no clue what was holding the path.
          if [ -e "$tt" ]; then
            echo "::error::$tt survived the pre-checkout clean"
            ls -lan "$tt" || true
            exit 1
          fi

      - uses: actions/checkout@v5
        with:
          clean: true

      - name: Decide whether the guides gate this publish
        id: plan
        run: |
          set -euo pipefail
          skip() { echo "guides=skip" >> "$GITHUB_OUTPUT"; echo "reason=$1" >> "$GITHUB_OUTPUT"; echo "guides skipped: $1"; }
          if [ "${{ needs.label.outputs.publish }}" != "true" ]; then
            skip "build-only: nothing is published"
          elif [ ! -f scripts/doc-guide/regen-manual.sh ]; then
            skip "this product has no guides (scripts/doc-guide/regen-manual.sh)"
          else
            decision="$(nix develop .#build --command bash -c '
              set -euo pipefail
              bun install --frozen-lockfile >/dev/null
              cli="$(bun -e "console.log(Bun.resolveSync(\"@norskvideo/ctl-dev-kit/publish/product-channel\", process.cwd()))")"
              bun "$cli" guide-gate
            ')"
            if [ "$decision" = "run" ]; then
              echo "guides=run" >> "$GITHUB_OUTPUT"
              # The engine tier runs in guides-engine, beside this gate.
              if jq -e '.scripts["docs:guides:engine"]' package.json >/dev/null; then
                echo "engine=run" >> "$GITHUB_OUTPUT"
              fi
            else
              skip "${decision#skip$'\t'}"
            fi
          fi

      - name: Write the Norsk license (from the org secret)
        if: ${{ steps.plan.outputs.guides == 'run' }}
        env:
          NORSK_LICENSE_V2: ${{ secrets.NORSK_LICENSE_V2 }}
        run: printf '%s' "$NORSK_LICENSE_V2" > "$RUNNER_TEMP/norsk-license.json"

      # Every norsk-ctl daemon starts the node MoQ relay (norsk-moq-relay) unless
      # told not to, and nothing removes it: it is a node singleton, and the reap
      # step only takes containers that mount this job's test-temp. So the first
      # test daemon on a shared runner left one, and every later one adopted it
      # (probe's smoke ran in relay mode on 2026-10-10; relay's smoke lost 4443
      # to it). This job's daemons run without the node relay and discovery, so
      # they neither start nor adopt one, and whatever is already on the host is
      # named here. It reports and never fails: the leftover may be someone's.
      - name: Isolate this job from node singletons other jobs left on this host
        run: |
          set -uo pipefail
          echo "NORSK_CTL_MOQ_RELAY_ENABLE=0" >> "$GITHUB_ENV"
          echo "NORSK_CTL_DISCOVERY_ENABLE=0" >> "$GITHUB_ENV"
          left="$(docker ps -a --filter label=norsk-ctl.role --format '{{.Names}} {{.Label "norsk-ctl.role"}} {{.Status}} (created {{.CreatedAt}})' 2>/dev/null || true)"
          if [ -z "$left" ]; then echo "no norsk-ctl containers on this host"; exit 0; fi
          echo "norsk-ctl containers already on this host (other jobs' running instances are expected):"
          echo "$left"
          while IFS= read -r line; do
            role="$(awk '{print $2}' <<<"$line")"
            case "$role" in
              moq-relay|discovery)
                echo "::warning title=node singleton left on this host::$line -- left by an earlier job (a daemon's auto-start, or a run killed before its reap). This job's daemons run without it and will not adopt it; remove it with docker rm -f if nothing on this host needs it." ;;
            esac
          done <<<"$left"

      # The norsk-ctl this commit pins (flake.nix ctlVersion, moved by
      # bump.yml's nightly PR), never S3's moving `latest`: a guide run is then
      # a real repeat, and a red belongs to a commit (ci-fragility proposal A).
      - name: Download the pinned norsk-ctl binary (flake.nix ctlVersion)
        if: ${{ steps.plan.outputs.guides == 'run' }}
        run: |
          set -euo pipefail
          S3="https://s3.eu-west-1.amazonaws.com/norsk.video/norsk-ctl"
          ver="$( { sed -nE 's/^[[:space:]]*ctlVersion = "([^"]+)";.*/\1/p' flake.nix 2>/dev/null || true; } | head -1)"
          [ -n "$ver" ] || { echo "::error::flake.nix pins no ctlVersion. CI runs the norsk-ctl this commit pins (the nightly bump.yml moves it); add ctlVersion = \"<version>\"; to flake.nix."; exit 1; }
          echo "norsk-ctl pinned by flake.nix ctlVersion -> $ver"
          curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$ver/norsk-ctl-$ver-linux-x64" -o "$RUNNER_TEMP/norsk-ctl"
          chmod +x "$RUNNER_TEMP/norsk-ctl"
          "$RUNNER_TEMP/norsk-ctl" --version || true

      # Cold-runner first pulls overrun the harness's per-test launch timeouts;
      # pre-pull the universal media + studio images (every product pins them in
      # manifest.seed.json) so compose/run hit local images. A product needing
      # more (e.g. a WHIP driver) pulls them in scripts/doc-guide/prepull-extra.sh.
      - name: Pre-pull the media + studio images
        if: ${{ steps.plan.outputs.guides == 'run' }}
        run: |
          set -euo pipefail
          # An sdk-app has no seed: its prepull-extra hook pulls its engine.
          if [ -f manifest.seed.json ]; then
            for img in "$(jq -r '.latest.media' manifest.seed.json)" "$(jq -r '.latest.studio' manifest.seed.json)"; do
              echo "pre-pulling $img"
              docker pull "$img"
            done
          fi
          if [ -x scripts/doc-guide/prepull-extra.sh ]; then
            echo "running scripts/doc-guide/prepull-extra.sh"
            ./scripts/doc-guide/prepull-extra.sh
          fi

      # The deterministic tiers (fixture and frontend: no daemon, no instances),
      # then the manual assembled from what they captured. These gate. The engine
      # tier runs on its own below: it stands up real instances, is known-flaky
      # on container DNS (norsk-ctl task 57), and reports without gating until
      # that is fixed (ci-workflows docs/dashboard/ci-fragility.md, proposal E).
      # regen-manual.sh owns the slug list and honours SKIP_ENGINE; it rebuilds
      # every workspace the manual depends on first, and with-display.sh gives
      # headless chromium an X display.
      - name: Regenerate the illustrated manual (deterministic tiers)
        if: ${{ steps.plan.outputs.guides == 'run' }}
        env:
          NORSK_CTL_BINARY: ${{ runner.temp }}/norsk-ctl
          NORSK_LICENSE_FILE: ${{ runner.temp }}/norsk-license.json
          # This runner launches Studio as a HOST sibling (DooD), so the harness
          # reaches host-published ports via the host-gateway alias, not
          # localhost — same as the integration suite.
          NORSK_TEST_HOST: host.docker.internal
          # Reach launched instances over norsk-net by service DNS (as the
          # integration tier already does) AND, on a ctl that supports it, launch
          # them binding no host ports — so concurrent doc-guide instances can't
          # collide on the product's deterministic host-port bands.
          NORSK_TEST_NET: direct
          SKIP_ENGINE: "1"
        run: |
          nix develop .#build --command bash -c '
            set -euo pipefail
            # Belt-and-braces now that the checkout cleans: a persisted node_modules
            # workspace dep whose version moved leaves stale copies that
            # --frozen-lockfile does not reliably relink, so a new export is "not
            # found". Nuke EVERY node_modules — including nested per-workspace ones
            # — for a deterministic install. A nightly can afford it.
            find . -name node_modules -type d -prune -exec rm -rf {} + 2>/dev/null || true
            bun install --frozen-lockfile
            bun run build:no-lint
            bash scripts/with-display.sh scripts/doc-guide/regen-manual.sh
          '

      - name: Reap the containers this job launched
        # Whatever the tests did. A teardown cut short by a test timeout leaves
        # its instance running on the shared Docker daemon (see JOB_REAP_STEP in
        # conventions/workflow-shell.ts); everything this job launched mounts its
        # own test-temp, so that is what goes.
        if: always()
        run: |
          base="${NORSK_CTL_TEST_TMP:-}"
          tt="$GITHUB_WORKSPACE/test-temp"
          s=""
          for c in $(docker ps -aq); do
            src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
            if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi
          done
          if [ -n "$s" ]; then
            echo "::warning::removing containers this job left running:$s"
            docker rm -f $s
          fi

  # The engine tier of the guides, beside the gate rather than in it (see the
  # deterministic step above): it runs whenever the gate runs the guides and the
  # product has an engine tier, and reports -- a warning, the run summary and
  # the build-image cell's detail -- but publish does not wait on it, and a
  # failure leaves the job green so bot-prs can still merge. To make it gate
  # again once norsk-ctl task 57 is fixed, add it to publish's needs and drop
  # continue-on-error from the engine step.
  guides-engine:
    needs: [label, gate-guides]
    # !cancelled(): a red deterministic gate must not hide the engine tier's
    # result. The plan's output is set before any guide runs.
    if: ${{ !cancelled() && needs.gate-guides.outputs.engine == 'run' }}
    runs-on: x64
    # One engine-tier run per repo at a time, nightly (publish-docs.yml) and
    # this alike: both launch the same instance ids on the x64 pool's shared
    # docker daemon, and ctl names each compose project after its instance id.
    # Queue, never cancel.
    concurrency:
      group: doc-guides-${{ github.repository }}
      cancel-in-progress: false
    outputs:
      result: ${{ steps.engine.outcome }}
    steps:
      # Runs BEFORE checkout, and does two things: points the harness's temp base
      # out of the workspace for every later step in this job, and clears whatever
      # the old repo-local default left behind on this runner. The body carries the
      # why. It fails the job only when legacy litter survives -- which would take
      # the checkout down a step later anyway, but report nothing useful.
      - name: Route test-temp out of the workspace (pre-checkout)
        run: |
          set -uo pipefail
          # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
          # HOST daemon has not seen before is created by dockerd, as ROOT, on the
          # host -- even when the launch passes --container-user (reproduced
          # 2026-09-04). The harness's default base is repo-local for an OrbStack
          # inotify constraint that binds macOS dev and nothing here, so every run
          # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
          # cannot remove. That took commentary's checks AND integration red on
          # 2026-09-03. Use the sanctioned override, per-run and outside the
          # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
          base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$base"
          echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
          # Yesterday's bases hold dockerd-created root dirs, so a non-root rm
          # EACCESes. They cannot wedge a checkout, but they do fill the disk.
          for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
            docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
          done
          # LEGACY: a runner that ran the old default still carries a workspace
          # test-temp, and a checkout with clean:true dies on it. Reap the holders
          # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
          # RESTARTING leaked container re-creates the path as root seconds after
          # any clean. Match on mount SOURCE, not a label: the holders are untracked
          # by construction, because a killed run labels nothing.
          tt="$GITHUB_WORKSPACE/test-temp"
          [ -d "$tt" ] || exit 0
          stuck=""
          for c in $(docker ps -aq 2>/dev/null || true); do
            if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
              stuck="$stuck $c"
            fi
          done
          if [ -n "$stuck" ]; then
            echo "reaping containers holding mounts under $tt:$stuck"
            docker rm -f $stuck || true
          fi
          # The DIRECTORY, not just its contents: an empty but root-owned base
          # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
          # container unlink the leaf.
          docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
            sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
          # Fail HERE if it survived. The checkout fails on it either way, but
          # reports only an EACCES rmdir with no clue what was holding the path.
          if [ -e "$tt" ]; then
            echo "::error::$tt survived the pre-checkout clean"
            ls -lan "$tt" || true
            exit 1
          fi

      - uses: actions/checkout@v5
        with:
          clean: true

      - name: Write the Norsk license (from the org secret)
        env:
          NORSK_LICENSE_V2: ${{ secrets.NORSK_LICENSE_V2 }}
        run: printf '%s' "$NORSK_LICENSE_V2" > "$RUNNER_TEMP/norsk-license.json"
      # Every norsk-ctl daemon starts the node MoQ relay (norsk-moq-relay) unless
      # told not to, and nothing removes it: it is a node singleton, and the reap
      # step only takes containers that mount this job's test-temp. So the first
      # test daemon on a shared runner left one, and every later one adopted it
      # (probe's smoke ran in relay mode on 2026-10-10; relay's smoke lost 4443
      # to it). This job's daemons run without the node relay and discovery, so
      # they neither start nor adopt one, and whatever is already on the host is
      # named here. It reports and never fails: the leftover may be someone's.
      - name: Isolate this job from node singletons other jobs left on this host
        run: |
          set -uo pipefail
          echo "NORSK_CTL_MOQ_RELAY_ENABLE=0" >> "$GITHUB_ENV"
          echo "NORSK_CTL_DISCOVERY_ENABLE=0" >> "$GITHUB_ENV"
          left="$(docker ps -a --filter label=norsk-ctl.role --format '{{.Names}} {{.Label "norsk-ctl.role"}} {{.Status}} (created {{.CreatedAt}})' 2>/dev/null || true)"
          if [ -z "$left" ]; then echo "no norsk-ctl containers on this host"; exit 0; fi
          echo "norsk-ctl containers already on this host (other jobs' running instances are expected):"
          echo "$left"
          while IFS= read -r line; do
            role="$(awk '{print $2}' <<<"$line")"
            case "$role" in
              moq-relay|discovery)
                echo "::warning title=node singleton left on this host::$line -- left by an earlier job (a daemon's auto-start, or a run killed before its reap). This job's daemons run without it and will not adopt it; remove it with docker rm -f if nothing on this host needs it." ;;
            esac
          done <<<"$left"

      # The norsk-ctl this commit pins (flake.nix ctlVersion, moved by
      # bump.yml's nightly PR), never S3's moving `latest`: a guide run is then
      # a real repeat, and a red belongs to a commit (ci-fragility proposal A).
      - name: Download the pinned norsk-ctl binary (flake.nix ctlVersion)
        run: |
          set -euo pipefail
          S3="https://s3.eu-west-1.amazonaws.com/norsk.video/norsk-ctl"
          ver="$( { sed -nE 's/^[[:space:]]*ctlVersion = "([^"]+)";.*/\1/p' flake.nix 2>/dev/null || true; } | head -1)"
          [ -n "$ver" ] || { echo "::error::flake.nix pins no ctlVersion. CI runs the norsk-ctl this commit pins (the nightly bump.yml moves it); add ctlVersion = \"<version>\"; to flake.nix."; exit 1; }
          echo "norsk-ctl pinned by flake.nix ctlVersion -> $ver"
          curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$ver/norsk-ctl-$ver-linux-x64" -o "$RUNNER_TEMP/norsk-ctl"
          chmod +x "$RUNNER_TEMP/norsk-ctl"
          "$RUNNER_TEMP/norsk-ctl" --version || true
      # Cold-runner first pulls overrun the harness's per-test launch timeouts;
      # pre-pull the universal media + studio images (every product pins them in
      # manifest.seed.json) so compose/run hit local images. A product needing
      # more (e.g. a WHIP driver) pulls them in scripts/doc-guide/prepull-extra.sh.
      - name: Pre-pull the media + studio images
        run: |
          set -euo pipefail
          # An sdk-app has no seed: its prepull-extra hook pulls its engine.
          if [ -f manifest.seed.json ]; then
            for img in "$(jq -r '.latest.media' manifest.seed.json)" "$(jq -r '.latest.studio' manifest.seed.json)"; do
              echo "pre-pulling $img"
              docker pull "$img"
            done
          fi
          if [ -x scripts/doc-guide/prepull-extra.sh ]; then
            echo "running scripts/doc-guide/prepull-extra.sh"
            ./scripts/doc-guide/prepull-extra.sh
          fi
      - name: Install and build the workspaces the guides drive
        run: |
          nix develop .#build --command bash -c '
            set -euo pipefail
            find . -name node_modules -type d -prune -exec rm -rf {} + 2>/dev/null || true
            bun install --frozen-lockfile
            bun run build:no-lint
          '

      # The engine tier: real instances, driven in a browser. Reported, not
      # gating (see the deterministic step). A non-zero exit is forgiven only
      # when the tier's own results say a teardown hook timed out after every
      # guide's test passed (doc-guide/engine-verdict.ts), as regen-manual.sh
      # forgives it. A product with no docs:guides:engine script has no engine
      # tier. Runs on the workspaces the step before it installed and built.
      - name: Run the engine-tier guides (reported, not gating)
        id: engine
        continue-on-error: true
        env:
          NORSK_CTL_BINARY: ${{ runner.temp }}/norsk-ctl
          NORSK_LICENSE_FILE: ${{ runner.temp }}/norsk-license.json
          # This runner launches Studio as a HOST sibling (DooD), so the harness
          # reaches host-published ports via the host-gateway alias, not
          # localhost — same as the integration suite.
          NORSK_TEST_HOST: host.docker.internal
          # Reach launched instances over norsk-net by service DNS (as the
          # integration tier already does) AND, on a ctl that supports it, launch
          # them binding no host ports — so concurrent doc-guide instances can't
          # collide on the product's deterministic host-port bands.
          NORSK_TEST_NET: direct
        run: |
          nix develop .#build --command bash -c '
            set -euo pipefail
            if ! jq -e ".scripts[\"docs:guides:engine\"]" package.json >/dev/null; then
              echo "no docs:guides:engine script: this product has no engine tier"
              exit 0
            fi
            export DOCS_GENERATE=1
            log="$(mktemp)"
            if bash scripts/with-display.sh bun run docs:guides:engine >"$log" 2>&1; then
              cat "$log"
            else
              cat "$log"
              verdict="$(bun -e "console.log(Bun.resolveSync(\"@norskvideo/ctl-dev-kit/doc-guide/engine-verdict\", process.cwd()))")"
              bun "$verdict" "$log"
            fi
          '

      # Said three ways, so a failure is never silent: a warning on the run, a
      # line in its summary, and the cell's detail (the notify job).
      - name: Report the engine tier (not gating)
        if: ${{ !cancelled() }}
        env:
          OUTCOME: ${{ steps.engine.outcome }}
        run: |
          case "$OUTCOME" in
            success) echo "engine-tier guides passed (or this product has none)" ;;
            failure)
              echo "::warning title=engine-tier guides failed (not gating)::the engine tier does not gate until norsk-ctl task 57 is fixed. See the engine step's log."
              printf '### Engine-tier guides: failed (not gating)\n\nThe deterministic guide tiers gate; the engine tier only reports until its container networking is fixed (norsk-ctl task 57). See the step log.\n' >> "$GITHUB_STEP_SUMMARY" ;;
            *) echo "engine tier: ${OUTCOME:-not run}" ;;
          esac

      - name: Reap the containers this job launched
        # Whatever the tests did. A teardown cut short by a test timeout leaves
        # its instance running on the shared Docker daemon (see JOB_REAP_STEP in
        # conventions/workflow-shell.ts); everything this job launched mounts its
        # own test-temp, so that is what goes.
        if: always()
        run: |
          base="${NORSK_CTL_TEST_TMP:-}"
          tt="$GITHUB_WORKSPACE/test-temp"
          s=""
          for c in $(docker ps -aq); do
            src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
            if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi
          done
          if [ -n "$s" ]; then
            echo "::warning::removing containers this job left running:$s"
            docker rm -f $s
          fi

  # Combine the per-arch images into a multi-arch manifest and write the two
  # published tags. Skipped for a build-only product (empty PUBLISH_IMAGE) and on
  # a dry run.
  publish:
    needs: [label, build, gate-checks, gate-guides]
    if: ${{ needs.label.outputs.publish == 'true' && !inputs.dry_run && github.event_name != 'pull_request' }}
    runs-on: x64
    steps:
      # Runs BEFORE checkout, and does two things: points the harness's temp base
      # out of the workspace for every later step in this job, and clears whatever
      # the old repo-local default left behind on this runner. The body carries the
      # why. It fails the job only when legacy litter survives -- which would take
      # the checkout down a step later anyway, but report nothing useful.
      - name: Route test-temp out of the workspace (pre-checkout)
        run: |
          set -uo pipefail
          # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
          # HOST daemon has not seen before is created by dockerd, as ROOT, on the
          # host -- even when the launch passes --container-user (reproduced
          # 2026-09-04). The harness's default base is repo-local for an OrbStack
          # inotify constraint that binds macOS dev and nothing here, so every run
          # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
          # cannot remove. That took commentary's checks AND integration red on
          # 2026-09-03. Use the sanctioned override, per-run and outside the
          # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
          base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$base"
          echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
          # Yesterday's bases hold dockerd-created root dirs, so a non-root rm
          # EACCESes. They cannot wedge a checkout, but they do fill the disk.
          for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
            docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
          done
          # LEGACY: a runner that ran the old default still carries a workspace
          # test-temp, and a checkout with clean:true dies on it. Reap the holders
          # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
          # RESTARTING leaked container re-creates the path as root seconds after
          # any clean. Match on mount SOURCE, not a label: the holders are untracked
          # by construction, because a killed run labels nothing.
          tt="$GITHUB_WORKSPACE/test-temp"
          [ -d "$tt" ] || exit 0
          stuck=""
          for c in $(docker ps -aq 2>/dev/null || true); do
            if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
              stuck="$stuck $c"
            fi
          done
          if [ -n "$stuck" ]; then
            echo "reaping containers holding mounts under $tt:$stuck"
            docker rm -f $stuck || true
          fi
          # The DIRECTORY, not just its contents: an empty but root-owned base
          # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
          # container unlink the leaf.
          docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
            sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
          # Fail HERE if it survived. The checkout fails on it either way, but
          # reports only an EACCES rmdir with no clue what was holding the path.
          if [ -e "$tt" ]; then
            echo "::error::$tt survived the pre-checkout clean"
            ls -lan "$tt" || true
            exit 1
          fi

      - uses: actions/checkout@v5
        with:
          clean: true

      - name: Log in to Docker Hub
        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
        with:
          username: norskvideo
          password: ${{ secrets.DOCKER_PAT }}

      - name: Create + push the multi-arch manifest (dated label + channel)
        run: |
          label="${{ needs.label.outputs.label }}"
          channel="${{ needs.label.outputs.channel }}"
          # buildx imagetools create, not `docker manifest create`: the arm64
          # runner's containerd image store pushes single-arch images wrapped in
          # an OCI index, which `docker manifest create` refuses to nest. imagetools
          # accepts index sources and writes every -t tag in one push.
          docker buildx imagetools create \
            -t "${PUBLISH_IMAGE}:${label}" \
            -t "${PUBLISH_IMAGE}:${channel}" \
            "${PUBLISH_IMAGE}:${label}-amd64" \
            "${PUBLISH_IMAGE}:${label}-arm64"
      - name: Reap the containers this job launched
        # Whatever the tests did. A teardown cut short by a test timeout leaves
        # its instance running on the shared Docker daemon (see JOB_REAP_STEP in
        # conventions/workflow-shell.ts); everything this job launched mounts its
        # own test-temp, so that is what goes.
        if: always()
        run: |
          base="${NORSK_CTL_TEST_TMP:-}"
          tt="$GITHUB_WORKSPACE/test-temp"
          s=""
          for c in $(docker ps -aq); do
            src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
            if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi
          done
          if [ -n "$s" ]; then
            echo "::warning::removing containers this job left running:$s"
            docker rm -f $s
          fi

  # Record the published image in this product's channels object
  # (s3://norsk.video/products/<product>/channels.json), which ctl reads to
  # register the product. Only reached after the gate passed, so a row is a
  # statement that the image passed it. The role is per product and can write
  # only its own prefix (norsk-ctl docs/ci/aws-oidc/apply-products.sh); until it
  # exists this warns and records nothing, rather than failing a publish.
  record:
    needs: [label, gate-guides, publish]
    if: ${{ needs.publish.result == 'success' && github.event_name != 'pull_request' }}
    runs-on: x64
    permissions:
      contents: read
      id-token: write
    steps:
      # Runs BEFORE checkout, and does two things: points the harness's temp base
      # out of the workspace for every later step in this job, and clears whatever
      # the old repo-local default left behind on this runner. The body carries the
      # why. It fails the job only when legacy litter survives -- which would take
      # the checkout down a step later anyway, but report nothing useful.
      - name: Route test-temp out of the workspace (pre-checkout)
        run: |
          set -uo pipefail
          # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
          # HOST daemon has not seen before is created by dockerd, as ROOT, on the
          # host -- even when the launch passes --container-user (reproduced
          # 2026-09-04). The harness's default base is repo-local for an OrbStack
          # inotify constraint that binds macOS dev and nothing here, so every run
          # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
          # cannot remove. That took commentary's checks AND integration red on
          # 2026-09-03. Use the sanctioned override, per-run and outside the
          # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
          base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$base"
          echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
          # Yesterday's bases hold dockerd-created root dirs, so a non-root rm
          # EACCESes. They cannot wedge a checkout, but they do fill the disk.
          for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
            docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
          done
          # LEGACY: a runner that ran the old default still carries a workspace
          # test-temp, and a checkout with clean:true dies on it. Reap the holders
          # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
          # RESTARTING leaked container re-creates the path as root seconds after
          # any clean. Match on mount SOURCE, not a label: the holders are untracked
          # by construction, because a killed run labels nothing.
          tt="$GITHUB_WORKSPACE/test-temp"
          [ -d "$tt" ] || exit 0
          stuck=""
          for c in $(docker ps -aq 2>/dev/null || true); do
            if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
              stuck="$stuck $c"
            fi
          done
          if [ -n "$stuck" ]; then
            echo "reaping containers holding mounts under $tt:$stuck"
            docker rm -f $stuck || true
          fi
          # The DIRECTORY, not just its contents: an empty but root-owned base
          # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
          # container unlink the leaf.
          docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
            sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
          # Fail HERE if it survived. The checkout fails on it either way, but
          # reports only an EACCES rmdir with no clue what was holding the path.
          if [ -e "$tt" ]; then
            echo "::error::$tt survived the pre-checkout clean"
            ls -lan "$tt" || true
            exit 1
          fi

      - uses: actions/checkout@v5
        with:
          clean: true

      - name: Warn that no channels role is configured
        if: ${{ vars.AWS_PRODUCT_ROLE_ARN == '' }}
        run: echo "::warning::AWS_PRODUCT_ROLE_ARN is not set, so this publish was not recorded for ctl (norsk-ctl docs/ci/aws-oidc/apply-products.sh)"

      - name: Configure AWS credentials (this product's channels role)
        if: ${{ vars.AWS_PRODUCT_ROLE_ARN != '' }}
        uses: aws-actions/configure-aws-credentials@v6
        with:
          role-to-assume: ${{ vars.AWS_PRODUCT_ROLE_ARN }}
          role-session-name: product-channels-${{ github.run_id }}
          aws-region: eu-west-1

      - name: Record the image in the product's channels object
        if: ${{ vars.AWS_PRODUCT_ROLE_ARN != '' }}
        env:
          LABEL: ${{ needs.label.outputs.label }}
          CHANNEL: ${{ needs.label.outputs.channel }}
          GUIDES: ${{ needs.gate-guides.outputs.guides }}
          SKIP_REASON: ${{ needs.gate-guides.outputs.reason }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          set -euo pipefail
          image="${PUBLISH_IMAGE}:${LABEL}"
          runtime="$(docker buildx imagetools inspect "$image" --format '{{ json (index .Image "linux/amd64") }}' \
            | jq -r '.config.Labels["norsk-ctl.runtime-images"] // ""')"
          aws() { nix shell --inputs-from . nixpkgs#awscli2 --command aws "$@"; }
          nix develop .#build --command bash -c 'bun install --frozen-lockfile >/dev/null'
          cli="$(nix develop .#build --command bun -e 'console.log(Bun.resolveSync("@norskvideo/ctl-dev-kit/publish/product-channel", process.cwd()))')"
          product="$(nix develop .#build --command bun "$cli" product --template licenseTemplate.json --image "$PUBLISH_IMAGE")"
          uri="s3://norsk.video/products/${product}/channels.json"
          current="$RUNNER_TEMP/channels.current.json"
          : > "$current"
          if aws s3api head-object --bucket norsk.video --key "products/${product}/channels.json" >/dev/null 2>&1; then
            aws s3 cp "$uri" "$current"
          fi
          skipped=()
          if [ "$GUIDES" = "skip" ]; then skipped=(--skipped "$SKIP_REASON"); fi
          nix develop .#build --command bun "$cli" row --current "$current" \
            --product "$product" --image-ref "$PUBLISH_IMAGE" --channel "$CHANNEL" \
            --image "$image" --sha "$GITHUB_SHA" --published-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            --runtime-images "$runtime" --run "$RUN_URL" "${skipped[@]}" > "$RUNNER_TEMP/channels.json"
          aws s3 cp "$RUNNER_TEMP/channels.json" "$uri" --content-type application/json --cache-control "max-age=60"
          echo "recorded $image as ${product}'s ${CHANNEL} -> $uri"
      - name: Reap the containers this job launched
        # Whatever the tests did. A teardown cut short by a test timeout leaves
        # its instance running on the shared Docker daemon (see JOB_REAP_STEP in
        # conventions/workflow-shell.ts); everything this job launched mounts its
        # own test-temp, so that is what goes.
        if: always()
        run: |
          base="${NORSK_CTL_TEST_TMP:-}"
          tt="$GITHUB_WORKSPACE/test-temp"
          s=""
          for c in $(docker ps -aq); do
            src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
            if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi
          done
          if [ -n "$s" ]; then
            echo "::warning::removing containers this job left running:$s"
            docker rm -f $s
          fi

  # Report this pipeline's result to the aggregated product CI dashboard
  # (id3as/ci-workflows) instead of posting its own pony -- the dashboard renders
  # the pony/emoji from the dispatched result. always() so a red or manual run
  # still reports.
  notify:
    needs: [
      label,
      build,
      gate-checks,
      gate-guides,
      guides-engine,
      publish,
      record
    ]
    if: ${{ !cancelled() && github.event_name != 'pull_request' }}
    runs-on: x64
    steps:
      # Runs BEFORE checkout, and does two things: points the harness's temp base
      # out of the workspace for every later step in this job, and clears whatever
      # the old repo-local default left behind on this runner. The body carries the
      # why. It fails the job only when legacy litter survives -- which would take
      # the checkout down a step later anyway, but report nothing useful.
      - name: Route test-temp out of the workspace (pre-checkout)
        run: |
          set -uo pipefail
          # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
          # HOST daemon has not seen before is created by dockerd, as ROOT, on the
          # host -- even when the launch passes --container-user (reproduced
          # 2026-09-04). The harness's default base is repo-local for an OrbStack
          # inotify constraint that binds macOS dev and nothing here, so every run
          # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
          # cannot remove. That took commentary's checks AND integration red on
          # 2026-09-03. Use the sanctioned override, per-run and outside the
          # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
          base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
          mkdir -p "$base"
          echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
          # Yesterday's bases hold dockerd-created root dirs, so a non-root rm
          # EACCESes. They cannot wedge a checkout, but they do fill the disk.
          for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
            docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
          done
          # LEGACY: a runner that ran the old default still carries a workspace
          # test-temp, and a checkout with clean:true dies on it. Reap the holders
          # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
          # RESTARTING leaked container re-creates the path as root seconds after
          # any clean. Match on mount SOURCE, not a label: the holders are untracked
          # by construction, because a killed run labels nothing.
          tt="$GITHUB_WORKSPACE/test-temp"
          [ -d "$tt" ] || exit 0
          stuck=""
          for c in $(docker ps -aq 2>/dev/null || true); do
            if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
              stuck="$stuck $c"
            fi
          done
          if [ -n "$stuck" ]; then
            echo "reaping containers holding mounts under $tt:$stuck"
            docker rm -f $stuck || true
          fi
          # The DIRECTORY, not just its contents: an empty but root-owned base
          # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
          # container unlink the leaf.
          docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
            sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
          # Fail HERE if it survived. The checkout fails on it either way, but
          # reports only an EACCES rmdir with no clue what was holding the path.
          if [ -e "$tt" ]; then
            echo "::error::$tt survived the pre-checkout clean"
            ls -lan "$tt" || true
            exit 1
          fi

      - uses: actions/checkout@v5
        with:
          clean: true
      # guides-engine reports in the detail below and never sets the state: a
      # failed engine tier is not a failed build, nor is one cancelled in its
      # concurrency queue.
      - id: meta
        env:
          NEEDS: ${{ toJSON(needs) }}
        run: |
          if jq -e 'del(.["guides-engine"]) | [.[].result] | any(. == "failure" or . == "cancelled")' <<<"$NEEDS" >/dev/null; then
            echo "state=failure" >> "$GITHUB_OUTPUT"
          else
            echo "state=success" >> "$GITHUB_OUTPUT"
          fi
      - uses: ./.github/actions/ci-status-dispatch
        with:
          token: ${{ secrets.CI_DISPATCH_TOKEN }}
          product: __PRODUCT__
          pipeline: build-image
          status: ${{ steps.meta.outputs.state }}
          # Only where something is published: on a build-only repo the gate has
          # nothing to gate, and saying so on every build is noise.
          # And when the engine-tier guides failed: they report, never gate.
          detail: "${{ needs.label.outputs.publish == 'true' && needs.gate-guides.outputs.guides == 'skip' && format('guides not gating: {0}', needs.gate-guides.outputs.reason) || needs.guides-engine.outputs.result == 'failure' && 'engine-tier guides failed (not gating; norsk-ctl task 57)' || '' }}"
      - name: Reap the containers this job launched
        # Whatever the tests did. A teardown cut short by a test timeout leaves
        # its instance running on the shared Docker daemon (see JOB_REAP_STEP in
        # conventions/workflow-shell.ts); everything this job launched mounts its
        # own test-temp, so that is what goes.
        if: always()
        run: |
          base="${NORSK_CTL_TEST_TMP:-}"
          tt="$GITHUB_WORKSPACE/test-temp"
          s=""
          for c in $(docker ps -aq); do
            src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
            if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi
          done
          if [ -n "$s" ]; then
            echo "::warning::removing containers this job left running:$s"
            docker rm -f $s
          fi