# ctl-dev-kit convention: integration # # The integration tier, single-sourced in @norskvideo/ctl-dev-kit # (conventions/integration.yml) and drift-checked — the two per-repo lines are # the ci-status-dispatch `product:` key and the `runner:` label array. The # marker on line 1 is what puts a copy under the gate: a repo converges by # replacing its hand-owned workflow with this file (product and runner filled # in) plus whatever hooks below its suite needs, proven by a workflow_dispatch # run on the branch. Until a copy carries the marker it is hand-owned and # ungated — which is the state fleet review 04c s3.1 found every product in. # # What the tier is: the product's slow suite (daemon + Docker + licence) run # against the RELEASED norsk-ctl binary, exactly as a customer would run it — a # split product repo has no ctl source. `bun run test:integration` is the # product's claim of what that suite is, and two things believe it: this # repo's nightly bump PR, which runs it on a new ctl pin and merges only when # green, and ctl's (advisory) release gate, whose ctl-candidate dispatch runs it # and shows the result on the CI board. # # Per-repo variation lives in four optional hooks, not in this file: # scripts/integration/prepull-extra.sh extra images the suite launches (a # driver, a sink, a sidecar) # scripts/integration/prepare.sh built before the suite runs (a # frontend dist the daemon fetches, an # image the workflow launches) # scripts/integration/verify.sh after the suite; absent, a product # with tests/demo.spec.ts gets # `bun run demo -- check` # scripts/integration/contention.json opts the suite into contention verdicts # (thresholds; {} takes the defaults): a # failure the starved HOST caused cancels # the run instead of failing it. See # contention-suite.sh and the steps below. # # The suite is piped into tee so its log survives as an artifact, and the step # sets pipefail FIRST: the default `bash -e {0}` has none, so without it the # step exits with tee's status and a dead suite reads green — both turnkeys # ran a void tier for weeks that way, feeding a green candidate result to the # release gate. tee-pipefail.test.ts guards the line; it ships with this file. # # Runner selection is a one-entry matrix carrying a JSON label array, because a # string like '[ARM64, nvidia]' in runs-on is ONE label, not two. Default # '["x64"]' — the docker-outside-of-docker pool, where launched containers are # host siblings and the harness reaches them by compose name on norsk-net (the # topology step below says so in NORSK_TEST_HOST/NORSK_TEST_NET); the GPU box # is '["ARM64","nvidia"]'. # # On a pull request the suite runs for a BOT branch only (bump/nightly, and the # retired bots' upgrade-latest/*, sync-dev-kit/* and chore/sync-ctl-packages, # which the PR that brings bump.yml rides on): a bump reaches main only once this # tier has passed on it, and bot-prs merges on that (ci-workflows # docs/dashboard/ci-fragility.md, proposal B). A human PR does not pay for the # slow tier; it runs on main after the merge, as before. A PR's result is the # PR's, so it is not reported as the product's integration cell. name: integration on: repository_dispatch: types: [ctl-candidate] push: branches: [main] pull_request: workflow_dispatch: inputs: ctl_channel: description: "norsk-ctl S3 channel to test against in place of the repo's pin (flake.nix ctlVersion). Empty runs the pin; release-gate dispatches 'rc' to report on a candidate (advisory: latest is not held on it)." type: string default: "" permissions: contents: read # A ctl-candidate validation must not be cancellable out of a result, because # release-collect scores anything other than a literal success as red — a # CANCELLED run included. (Until the gate went advisory on 2026-10-17 it also # answered a red with one re-dispatch, which is what made this bite.) # # Keying on event_name alone was not enough: every candidate is the SAME event, # so they shared a group and cancelled each other, and the re-dispatch cancelled # the very run it was asking for. On 2026-09-05 commentary was dispatched four # times in thirteen minutes, every run cancelled before it could start on the # backlogged pool, and rc-gate/0.1.0-2026-09-05-98e09d1/FAILED was written for a # candidate on which nothing had actually failed — while playout, studio and # reuters, fast enough to finish inside the window, all went green. # # So: separate the candidates (client_payload.version — empty on push/PR, which # leaves those groups exactly as they were), and do not cancel a dispatch run. # A superseded PENDING run is dropped before it starts, fires no # report-candidate, and scores nothing — which is why queueing is safe here # where cancelling is not. Nothing cancels a run in progress # (no-cancel-in-progress.test.ts): a burst of pushes queues and collapses to # the newest pending run. # # Trade-off, stated: two candidates' validations can now overlap in one repo, # and their tiers derive host ports from the same per-slug bands. The bands were # moved out of the kernel's ephemeral range (ctl-test-harness 0.1.42); an # ephemeral daemon port is the durable answer and is not wired yet. concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}-${{ github.event.client_payload.version }} cancel-in-progress: false jobs: integration: if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && (github.head_ref == 'bump/nightly' || startsWith(github.head_ref, 'upgrade-latest/') || startsWith(github.head_ref, 'sync-dev-kit/') || github.head_ref == 'chore/sync-ctl-packages')) }} strategy: matrix: include: - runner: '["x64"]' runs-on: ${{ fromJSON(matrix.runner) }} # actions: write is for one call only: the last step cancelling this run # when the host was too contended to judge it. permissions: contents: read actions: write steps: # Runs BEFORE checkout, and does two things: points the harness's temp base # out of the workspace for every later step in this job, and clears whatever # the old repo-local default left behind on this runner. The body carries the # why. It fails the job only when legacy litter survives -- which would take # the checkout down a step later anyway, but report nothing useful. - name: Route test-temp out of the workspace (pre-checkout) run: | set -uo pipefail # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the # HOST daemon has not seen before is created by dockerd, as ROOT, on the # host -- even when the launch passes --container-user (reproduced # 2026-09-04). The harness's default base is repo-local for an OrbStack # inotify constraint that binds macOS dev and nothing here, so every run # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx` # cannot remove. That took commentary's checks AND integration red on # 2026-09-03. Use the sanctioned override, per-run and outside the # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped. base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" mkdir -p "$base" echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV" # Yesterday's bases hold dockerd-created root dirs, so a non-root rm # EACCESes. They cannot wedge a checkout, but they do fill the disk. for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true done # LEGACY: a runner that ran the old default still carries a workspace # test-temp, and a checkout with clean:true dies on it. Reap the holders # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a # RESTARTING leaked container re-creates the path as root seconds after # any clean. Match on mount SOURCE, not a label: the holders are untracked # by construction, because a killed run labels nothing. tt="$GITHUB_WORKSPACE/test-temp" [ -d "$tt" ] || exit 0 stuck="" for c in $(docker ps -aq 2>/dev/null || true); do if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then stuck="$stuck $c" fi done if [ -n "$stuck" ]; then echo "reaping containers holding mounts under $tt:$stuck" docker rm -f $stuck || true fi # The DIRECTORY, not just its contents: an empty but root-owned base # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root # container unlink the leaf. docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \ sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true # Fail HERE if it survived. The checkout fails on it either way, but # reports only an EACCES rmdir with no clue what was holding the path. if [ -e "$tt" ]; then echo "::error::$tt survived the pre-checkout clean" ls -lan "$tt" || true exit 1 fi - uses: actions/checkout@v5 with: clean: true # A containerised runner (docker-outside-of-docker) reaches host-published # ports via the host-gateway alias and can join norsk-net for direct reach; # a bare-metal host runner (the GPU box) must stay on localhost + publish. # Detect rather than hardcode, so one byte-identical file serves both. # NORSK_CTL_PRODUCT_REACH rides along because this step has already # established the fact the daemon would otherwise sniff from its own # cgroup: `product add --image` loopback-publishes the control plane and # polls it for readiness, and on a DooD runner that loopback is the docker # HOST's, so registration times out at 60s with nothing in the log saying # why. The daemon's own detection stays the default; here the probe that # already knows simply says so. - name: Probe runner docker topology and set NORSK_TEST_HOST + NORSK_TEST_NET run: | set -uo pipefail echo "hostname=$(hostname)" if docker inspect "$(hostname)" >/dev/null 2>&1; then echo "TOPO: hostname resolves to a container -> runner is containerized (DooD)" echo "NORSK_TEST_HOST=host.docker.internal" >> "$GITHUB_ENV" echo "NORSK_TEST_NET=direct" >> "$GITHUB_ENV" echo "NORSK_CTL_PRODUCT_REACH=container-address" >> "$GITHUB_ENV" echo "NORSK_RUNNER_OWNS_DOCKER=false" >> "$GITHUB_ENV" else echo "TOPO: hostname is not a container -> runner is host-based" echo "NORSK_TEST_HOST=localhost" >> "$GITHUB_ENV" echo "NORSK_TEST_NET=publish" >> "$GITHUB_ENV" echo "NORSK_CTL_PRODUCT_REACH=host" >> "$GITHUB_ENV" echo "NORSK_RUNNER_OWNS_DOCKER=true" >> "$GITHUB_ENV" fi # Contention verdicts (scripts/integration/contention.json) read host # pressure from Linux PSI, and whether a runner can is a property of its # kernel (CONFIG_PSI, the psi= boot flag) and of its container (cgroup v2, # what is masked). This says which, on every run and whether or not the # repo has opted in, so the log answers "does the signal exist on this # host?" before anything depends on it. It only reports. - name: Probe whether this runner can read host pressure (PSI) run: | set -uo pipefail echo "host=$(hostname) kernel=$(uname -r) cpus=$(nproc) of $(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo '?')" echo "loadavg: $(cat /proc/loadavg 2>/dev/null || echo unreadable)" grep -E '^(MemTotal|MemAvailable):' /proc/meminfo 2>/dev/null || true readable="" for f in /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io /sys/fs/cgroup/cpu.pressure /sys/fs/cgroup/memory.pressure; do if out="$(cat "$f" 2>&1)" && grep -q '^some avg10=' <<<"$out"; then echo "$f: readable" sed 's/^/ /' <<<"$out" readable="$readable $f" else echo "$f: NOT readable (${out:-empty})" fi done if [ -f /sys/fs/cgroup/cgroup.controllers ]; then echo "cgroup v2 at /sys/fs/cgroup; this process: $(cat /proc/self/cgroup 2>/dev/null)"; else echo "no cgroup v2 at /sys/fs/cgroup"; fi case "$readable" in *"/proc/pressure/cpu"*"/proc/pressure/memory"*) verdict="host-wide PSI readable (/proc/pressure)" ;; *"/sys/fs/cgroup/cpu.pressure"*"/sys/fs/cgroup/memory.pressure"*) verdict="only this cgroup's PSI readable: blind to the engine containers, which are host siblings" ;; *) verdict="PSI unavailable: contention verdicts would change nothing here" ;; esac if [ -f scripts/integration/contention.json ]; then optin="on (scripts/integration/contention.json)"; else optin="off (no scripts/integration/contention.json)"; fi echo "PSI on $(hostname): $verdict; contention verdicts $optin" echo "PSI on \`$(hostname)\`: $verdict; contention verdicts $optin" >> "$GITHUB_STEP_SUMMARY" # Integration needs this host's docker to be able to create a network: the # harness makes norsk-net and a network per instance. On 2026-10-02 # ashton-nuc's filter table had lost DOCKER-ISOLATION-STAGE-2, so every # create failed with "Chain 'DOCKER-ISOLATION-STAGE-2' does not exist" -- # and commentary, probe and playout each ran a FULL integration suite on it # and reported the result as test failures, 18 and 72 chain errors deep in # their logs. The same host took four of that night's norsk image jobs down # the same way. Nothing said "this machine's docker is broken". # # A containerised (DooD) runner must also be able to JOIN a new network: # the harness attaches it to each instance's network to reach the media # container. On 2026-10-07/08 ashton-nuc's runner container had lost its # network sandbox files (it lived through a dockerd restart), so every # `docker network connect` failed with "open .../resolv.conf: no such file # or directory" while create still worked: commentary saw "Unable to # connect" in every scenario, playout "could not join 's # network", and probe -- which never joins -- passed on the same host. # # The chain is not readable without root on these runners, so this asserts # the capabilities instead: create a throwaway network, join it when # containerised, leave, remove it. One step, its own name, so the jobs API # attributes the failure to it rather than to whatever composite step # happened to be running. - name: Check this host's docker can still create a network and attach the runner to it run: | set -uo pipefail net="preflight-${{ github.run_id }}-${{ github.run_attempt }}-$RANDOM" if ! err=$(docker network create "$net" 2>&1); then echo "::error::docker on this runner cannot create a network, so every container this suite starts would silently not exist -- the tests would fail for a reason that is not theirs. Host: $(hostname). Fix: \`sudo systemctl restart docker\` on that machine (it rebuilds the DOCKER/DOCKER-ISOLATION iptables chains), then re-run." echo "$err" echo "--- docker0 (a subnet other than 172.17.0.1/16 means it was re-addressed) ---" ip -br addr show docker0 2>/dev/null || true echo "--- networks and their subnets ---" for n in $(docker network ls --format '{{.Name}}' 2>/dev/null); do printf '%-28s ' "$n" docker network inspect "$n" --format '{{range .IPAM.Config}}{{.Subnet}} {{end}}' 2>/dev/null || echo done exit 1 fi me="$(hostname)" if docker inspect "$me" >/dev/null 2>&1; then # Containerised: join the network exactly as the harness will. if ! err=$(docker network connect "$net" "$me" 2>&1); then docker network disconnect -f "$net" "$me" >/dev/null 2>&1 || true docker network rm "$net" >/dev/null 2>&1 || true name="$(docker inspect -f '{{.Name}}' "$me" 2>/dev/null | sed 's|^/||')" echo "::error::this runner's container cannot join a new docker network, so the harness cannot reach any instance it starts -- the tests would fail as 'Unable to connect' / 'could not join', for a reason that is not theirs. Host: $(hostname) (container ${name:-$me}). Fix: on the docker host, \`docker restart ${name:-$me}\` once it is idle (it rebuilds the container's network sandbox), then re-run. ashton-nuc does this itself every 15 min (~steve/bin/runner-netcheck.sh, user timer runner-netcheck.timer)." echo "$err" exit 1 fi docker network disconnect "$net" "$me" >/dev/null 2>&1 || true echo "the runner container ($me) can join a new network -- ok" fi docker network rm "$net" >/dev/null 2>&1 || true echo "docker can create networks on $(hostname) -- ok" # `product add --image` publishes the control plane on 127.0.0.1:14321. The # daemon means to reap it on shutdown (daemon.ts, model B) but only does so # gracefully, and this workflow cancels its own superseded runs -- a killed # daemon reaps nothing and the port stays held for every later job here. # Kill by PORT rather than by name, prune the endpoints a removal can # strand, and then PROVE the port binds instead of assuming it. Only where # the docker daemon is this runner's own and jobs are serial; the GPU box # runs several runners as sibling CONTAINERS against one host daemon and is # excluded -- there, a blanket reap would kill a concurrent job. - name: Reap a stranded product control plane if: ${{ env.NORSK_RUNNER_OWNS_DOCKER == 'true' }} run: | set -uo pipefail port=14321 bindable() { docker run --rm -p "127.0.0.1:$port:1" alpine true >/dev/null 2>&1; } held="$(docker ps -a --filter "publish=$port" --format '{{.Names}} {{.Status}} {{.Image}}')" if [ -n "$held" ]; then echo "::warning::$port was held on entry by: $held" docker rm -f $(docker ps -aq --filter "publish=$port") || true fi leftover="$(docker ps -aq --filter 'name=^norsk-product-')" [ -z "$leftover" ] || docker rm -f $leftover || true if ! bindable; then echo "::warning::$port still not bindable after removing containers -- pruning dangling endpoints" docker network prune -f || true fi if ! bindable; then echo "::error::cannot bind 127.0.0.1:$port; the suite would fail later and more obscurely" docker ps -a --filter "publish=$port" || true exit 1 fi echo "$port is bindable" - name: Write the Norsk license (from the org secret) env: NORSK_LICENSE_V2: ${{ secrets.NORSK_LICENSE_V2 }} run: printf '%s' "$NORSK_LICENSE_V2" > "$RUNNER_TEMP/norsk-license.json" # Every norsk-ctl daemon starts the node MoQ relay (norsk-moq-relay) unless # told not to, and nothing removes it: it is a node singleton, and the reap # step only takes containers that mount this job's test-temp. So the first # test daemon on a shared runner left one, and every later one adopted it # (probe's smoke ran in relay mode on 2026-10-10; relay's smoke lost 4443 # to it). This job's daemons run without the node relay and discovery, so # they neither start nor adopt one, and whatever is already on the host is # named here. It reports and never fails: the leftover may be someone's. - name: Isolate this job from node singletons other jobs left on this host run: | set -uo pipefail echo "NORSK_CTL_MOQ_RELAY_ENABLE=0" >> "$GITHUB_ENV" echo "NORSK_CTL_DISCOVERY_ENABLE=0" >> "$GITHUB_ENV" left="$(docker ps -a --filter label=norsk-ctl.role --format '{{.Names}} {{.Label "norsk-ctl.role"}} {{.Status}} (created {{.CreatedAt}})' 2>/dev/null || true)" if [ -z "$left" ]; then echo "no norsk-ctl containers on this host"; exit 0; fi echo "norsk-ctl containers already on this host (other jobs' running instances are expected):" echo "$left" while IFS= read -r line; do role="$(awk '{print $2}' <<<"$line")" case "$role" in moq-relay|discovery) echo "::warning title=node singleton left on this host::$line -- left by an earlier job (a daemon's auto-start, or a run killed before its reap). This job's daemons run without it and will not adopt it; remove it with docker rm -f if nothing on this host needs it." ;; esac done <<<"$left" # Pick the binary for THIS runner's architecture: the GPU box is aarch64, # the x64 pool is x86_64, and the wrong one dies with "Exec format error" # deep in the suite. No `|| true` on --version: a bad binary must fail here. # # WHICH binary: the version this commit pins, flake.nix ctlVersion -- the # daemon the dev shell launches, moved by bump.yml's nightly PR. It used # to be S3's moving `latest`, read at run time, so a result could change # with no product commit and a re-run was never a real repeat (funke went # red and green again on 2026-10-09 without a commit either time; # ci-workflows docs/dashboard/ci-fragility.md, proposal A). A ctl-candidate # dispatch still tests its payload's channel (rc) -- that is what it is # for -- and a by-hand ctl_channel may too. ctl-pinned-in-repo.test.ts. - name: Download the pinned norsk-ctl binary env: CTL_CHANNEL: ${{ github.event.client_payload.channel || inputs.ctl_channel }} run: | set -euo pipefail case "$(uname -m)" in x86_64) arch=linux-x64 ;; aarch64 | arm64) arch=linux-arm64 ;; *) echo "unsupported runner arch: $(uname -m)" >&2; exit 1 ;; esac S3="https://s3.eu-west-1.amazonaws.com/norsk.video/norsk-ctl" if [ -n "${CTL_CHANNEL:-}" ]; then ver="$(curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$CTL_CHANNEL")" echo "norsk-ctl $CTL_CHANNEL channel -> $ver ($arch), in place of the repo pin" else ver="$( { sed -nE 's/^[[:space:]]*ctlVersion = "([^"]+)";.*/\1/p' flake.nix 2>/dev/null || true; } | head -1)" [ -n "$ver" ] || { echo "::error::flake.nix pins no ctlVersion. CI runs the norsk-ctl this commit pins (the nightly bump.yml moves it); add ctlVersion = \"\"; to flake.nix."; exit 1; } echo "norsk-ctl pinned by flake.nix ctlVersion -> $ver ($arch)" fi curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$ver/norsk-ctl-$ver-$arch" -o "$RUNNER_TEMP/norsk-ctl" chmod +x "$RUNNER_TEMP/norsk-ctl" "$RUNNER_TEMP/norsk-ctl" --version # Everything below runs dev-kit and test-harness code out of node_modules, # and `clean: true` on the checkout means this job starts without any. - name: Install dependencies run: nix develop .#build --command bun install --frozen-lockfile # Before anything is built or launched: does this licence actually name # this product? `product add` refuses one it does not, and that refusal # reaches the log as a 60-second readiness timeout indistinguishable from # a product that failed to start. reuters ran four "successful" hourly # tiers dead at exactly that refusal (04c s3.1). # # The product name is the repo's own PRODUCT_NAME, not the short board key # above (`funke` vs `funke-pegasus`): the licence names the product. - name: Check the licence entitles this product run: | set -euo pipefail product="$(sed -nE 's/.*PRODUCT_NAME[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' shared/src/version.ts | head -1)" [ -n "$product" ] || { echo "::error::could not read PRODUCT_NAME from shared/src/version.ts"; exit 1; } nix develop .#build --command bun \ node_modules/@norskvideo/ctl-dev-kit/licence/check-entitlement.ts \ "$RUNNER_TEMP/norsk-license.json" "$product" # Instance launch docker-composes the media + studio images; on a cold # runner the first pull overruns the per-test timeout. Pre-pull them (tags # from manifest.seed.json), plus whatever the repo lists in the optional # scripts/integration/prepull-extra.sh hook (a driver image, a sink). - name: Pre-pull the media + studio images run: | set -euo pipefail # An sdk-app has no seed: its prepull-extra hook pulls its engine. if [ -f manifest.seed.json ]; then for img in "$(jq -r '.latest.media' manifest.seed.json)" "$(jq -r '.latest.studio' manifest.seed.json)"; do echo "pre-pulling $img" docker pull "$img" done fi if [ -x scripts/integration/prepull-extra.sh ]; then echo "running scripts/integration/prepull-extra.sh" ./scripts/integration/prepull-extra.sh fi # The launched-instances manifest is what the reap step below reads: the # harness appends every instance id it launches, so a run that dies # mid-suite still names what it left behind. - name: Run integration suite env: NORSK_CTL_BINARY: ${{ runner.temp }}/norsk-ctl NORSK_LICENSE_FILE: ${{ runner.temp }}/norsk-license.json NORSK_LAUNCHED_MANIFEST: ${{ runner.temp }}/norsk-launched-instances # Where the harness copies each instance's engine logs and — since # ctl-test-harness 0.1.60 — its CONTAINER logs, before teardown removes # the containers. The studio container's stdout is the only place # "Refusing to run workflow" and the runner's own startup appear, and # every gate failure that pointed at `docker logs -studio-1` # was pointing at something already destroyed. # # Keyed on run id + attempt because these runners are SHARED between # repos: a bare $RUNNER_TEMP/norsk-logs is one directory nothing clears, # and the first artifact built that way carried ten of another product's # instances alongside its own three. Same shape the test-temp base uses. NORSK_CI_LOG_DIR: ${{ runner.temp }}/norsk-logs-${{ github.run_id }}-${{ github.run_attempt }} # Where contention-suite.sh puts each attempt's pressure timeline, # verdict and summary, keyed like the log dir for the same reason. NORSK_CONTENTION_DIR: ${{ runner.temp }}/contention-${{ github.run_id }}-${{ github.run_attempt }} run: | set -euo pipefail suite() { nix develop .#build --command bash -c ' set -euo pipefail # A frozen install does not reliably relink a hoisted package whose # version moved: a stale test-harness once survived a bump and a new # export was "not found" at import. Drop the harness copy so the # pinned version runs. rm -rf node_modules/@norskvideo/ctl-test-harness bun install --frozen-lockfile if [ -x scripts/integration/prepare.sh ]; then echo "running scripts/integration/prepare.sh" ./scripts/integration/prepare.sh fi if [ -f scripts/integration/contention.json ]; then bash node_modules/@norskvideo/ctl-dev-kit/conventions/contention-suite.sh else bun run test:integration fi if [ -x scripts/integration/verify.sh ]; then echo "running scripts/integration/verify.sh" ./scripts/integration/verify.sh elif [ -f tests/demo.spec.ts ]; then bun run demo -- check fi ' } # One suite per HOST. A host's runners are sibling containers on one # docker daemon, and the media containers a suite launches are not # confined to its runner's cpuset, so two suites side by side starve # each other's engines: on 2026-10-15/16 commentary's timing tests # failed in 10 of the 12 test2 runs that overlapped another suite on the # other runner, mostly commentary against itself, which a per-repo # concurrency group cannot stop without dropping a queued candidate. # /srv/ci-scratch is bind-mounted at that path into every runner on a # jankings host, so an flock there is the host's mutex; the kernel # drops it when this shell exits, however the job ends. Children close # fd 9 so a straggler cannot hold the host after the step. After an # hour the suite runs anyway, warned, rather than miss its result. # integration-host-lock.test.ts. if [ -d /srv/ci-scratch ] && [ -w /srv/ci-scratch ]; then lock=/srv/ci-scratch/locks/norsk-ctl-integration.lock mkdir -p "${lock%/*}" exec 9>>"$lock" echo "waiting for this host's integration lock ($lock)" waited=$SECONDS if flock -w 3600 9; then echo "holding this host's integration lock after $((SECONDS - waited))s" else echo "::warning title=integration ran unserialised::waited an hour for $lock on $(hostname); timing-sensitive tests may fail for a reason that is not theirs" fi else echo "no writable /srv/ci-scratch on this runner: the suite runs without the host lock" fi set +e suite 9>&- 2>&1 | tee "$RUNNER_TEMP/integration.log" rc=$? set -e # 75 is contention-suite.sh saying "uncertain": every failure # overlapped host contention, or the suite was stopped because the # host was starved. Believed only with the verdict file that says so, # since a suite could exit 75 for its own reasons. Not a failure: # the last step of this job cancels the run instead, so it reads # cancelled (not red) on the board, and bot-prs re-runs it later # rather than merging it or reporting it red. # Every other status, a real failure on a quiet box included, exits # exactly as it did. The %2C is a comma: workflow-command properties # are comma separated, so a bare one cut the title short. verdict="$NORSK_CONTENTION_DIR/verdict.json" if [ "$rc" -eq 75 ] && [ "$(jq -r .status "$verdict" 2>/dev/null)" = uncertain ]; then echo "::warning title=integration uncertain%2C the host was contended::$(jq -r .reason "$verdict"). This run will be cancelled rather than failed; the pressure timeline and per-test classification are in the job summary and the integration artifact." # The marker bot-prs reads off this job's annotations to tell this # cancel from any other (bot-prs-contended.test.ts). echo "::notice title=ctl-contention-uncertain::cancelled as uncertain, not failed: bot-prs re-runs it once the host may be quieter" echo "NORSK_CONTENTION_UNCERTAIN=1" >> "$GITHUB_ENV" exit 0 fi exit "$rc" # The log AND the harness evidence. Playwright and the harness write # screenshots, traces and captured container logs under test-results/; # a workflow that uploads only its stdout leaves the reader to re-run the # tier for evidence that already existed on the runner. - name: Upload the run log and test evidence if: always() uses: actions/upload-artifact@v4 with: name: integration-attempt-${{ github.run_attempt }} path: | ${{ runner.temp }}/integration.log test-results/ ${{ runner.temp }}/norsk-logs-${{ github.run_id }}-${{ github.run_attempt }} ${{ runner.temp }}/contention-${{ github.run_id }}-${{ github.run_attempt }} retention-days: 7 if-no-files-found: warn - name: Reap this run's leftover instance containers if: always() env: NORSK_LAUNCHED_MANIFEST: ${{ runner.temp }}/norsk-launched-instances run: | set -uo pipefail m="$NORSK_LAUNCHED_MANIFEST" [ -f "$m" ] || { echo "no launched-instances manifest -- nothing to reap"; exit 0; } sort -u "$m" | while IFS= read -r id; do [ -n "$id" ] || continue ids="$(docker ps -aq --filter "label=norsk-ctl.instance=$id" || true)" if [ -n "$ids" ]; then echo "reaping leftover instance '$id': $ids" docker rm -f $ids || true fi done rm -f "$m" || true - name: Reap the containers this job launched # Whatever the tests did. A teardown cut short by a test timeout leaves # its instance running on the shared Docker daemon (see JOB_REAP_STEP in # conventions/workflow-shell.ts); everything this job launched mounts its # own test-temp, so that is what goes. if: always() run: | base="${NORSK_CTL_TEST_TMP:-}" tt="$GITHUB_WORKSPACE/test-temp" s="" for c in $(docker ps -aq); do src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi done if [ -n "$s" ]; then echo "::warning::removing containers this job left running:$s" docker rm -f $s fi # Last, so the evidence is uploaded and the host reaped first. The suite # step flagged this run uncertain: its failures were the starved host's, # not the product's, so it is cancelled rather than failed. A cancelled # run is not reported to the CI board (notify is !cancelled()), and # bot-prs merges only a success, so a bump cannot land on it (it re-runs # a run whose job carries the suite step's ctl-contention-uncertain notice). # Cancellation is asynchronous: the step holds the job until it lands, # and if it is refused or never lands the job FAILS, because an uncertain # run must never finish reading green. contention-wired.test.ts. - name: Cancel this run, the host was too contended to judge it if: ${{ always() && env.NORSK_CONTENTION_UNCERTAIN == '1' }} env: GH_TOKEN: ${{ github.token }} run: | set -uo pipefail code="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ -H "Authorization: Bearer $GH_TOKEN" \ -H "Accept: application/vnd.github+json" \ "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/cancel")" [ "$code" = "202" ] || { echo "::error::could not cancel this uncertain run (HTTP $code), so it fails instead: read the contention summary before treating it as a product failure"; exit 1; } echo "cancel accepted; waiting for it to land" sleep 300 echo "::error::still running five minutes after the cancel was accepted, so it fails instead: read the contention summary before treating it as a product failure" exit 1 # Report this pipeline's result to the aggregated product CI dashboard # (id3as/ci-workflows). !cancelled() so a real pass/fail reports, but a # cancelled/superseded run does NOT — a cancellation dispatched as failure # would false-red the board. Candidate validations report to ctl instead. notify: needs: integration if: ${{ !cancelled() && github.event_name != 'repository_dispatch' && github.event_name != 'pull_request' }} runs-on: x64 steps: # Runs BEFORE checkout, and does two things: points the harness's temp base # out of the workspace for every later step in this job, and clears whatever # the old repo-local default left behind on this runner. The body carries the # why. It fails the job only when legacy litter survives -- which would take # the checkout down a step later anyway, but report nothing useful. - name: Route test-temp out of the workspace (pre-checkout) run: | set -uo pipefail # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the # HOST daemon has not seen before is created by dockerd, as ROOT, on the # host -- even when the launch passes --container-user (reproduced # 2026-09-04). The harness's default base is repo-local for an OrbStack # inotify constraint that binds macOS dev and nothing here, so every run # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx` # cannot remove. That took commentary's checks AND integration red on # 2026-09-03. Use the sanctioned override, per-run and outside the # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped. base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" mkdir -p "$base" echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV" # Yesterday's bases hold dockerd-created root dirs, so a non-root rm # EACCESes. They cannot wedge a checkout, but they do fill the disk. for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true done # LEGACY: a runner that ran the old default still carries a workspace # test-temp, and a checkout with clean:true dies on it. Reap the holders # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a # RESTARTING leaked container re-creates the path as root seconds after # any clean. Match on mount SOURCE, not a label: the holders are untracked # by construction, because a killed run labels nothing. tt="$GITHUB_WORKSPACE/test-temp" [ -d "$tt" ] || exit 0 stuck="" for c in $(docker ps -aq 2>/dev/null || true); do if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then stuck="$stuck $c" fi done if [ -n "$stuck" ]; then echo "reaping containers holding mounts under $tt:$stuck" docker rm -f $stuck || true fi # The DIRECTORY, not just its contents: an empty but root-owned base # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root # container unlink the leaf. docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \ sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true # Fail HERE if it survived. The checkout fails on it either way, but # reports only an EACCES rmdir with no clue what was holding the path. if [ -e "$tt" ]; then echo "::error::$tt survived the pre-checkout clean" ls -lan "$tt" || true exit 1 fi - uses: actions/checkout@v5 with: clean: true - id: meta run: | if [ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" = "true" ]; then echo "state=failure" >> "$GITHUB_OUTPUT" else echo "state=success" >> "$GITHUB_OUTPUT" fi - uses: ./.github/actions/ci-status-dispatch with: token: ${{ secrets.CI_DISPATCH_TOKEN }} product: __PRODUCT__ pipeline: integration status: ${{ steps.meta.outputs.state }} - name: Reap the containers this job launched # Whatever the tests did. A teardown cut short by a test timeout leaves # its instance running on the shared Docker daemon (see JOB_REAP_STEP in # conventions/workflow-shell.ts); everything this job launched mounts its # own test-temp, so that is what goes. if: always() run: | base="${NORSK_CTL_TEST_TMP:-}" tt="$GITHUB_WORKSPACE/test-temp" s="" for c in $(docker ps -aq); do src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi done if [ -n "$s" ]; then echo "::warning::removing containers this job left running:$s" docker rm -f $s fi # Report the rc-candidate validation result to the norsk-ctl release gate. Only # on a ctl-candidate dispatch; release-collect records it for the CI board. # Advisory: ctl promotes latest without waiting on it, and this repo takes a # new ctl only through its bump PR, which runs this tier on the new pin. report-candidate: needs: integration if: ${{ always() && github.event_name == 'repository_dispatch' && github.event.action == 'ctl-candidate' }} runs-on: x64 steps: - name: Fire ctl-candidate-result at norsk-ctl env: GH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }} # Lowercase on purpose: the drift gate masks every `product:` line, so # one per-repo key serves both the notify and the report jobs. product: __PRODUCT__ VERSION: ${{ github.event.client_payload.version }} RESULT: ${{ needs.integration.result }} run: | set -euo pipefail state=failure; [ "$RESULT" = "success" ] && state=success # `tier: integration`, named rather than defaulted: the release gate # tallies per tier, and this is the one that launches instances # (docs/release-gate.md). body="$(jq -nc --arg p "$product" --arg v "$VERSION" --arg r "$state" \ '{event_type:"ctl-candidate-result",client_payload:{product:$p,version:$v,result:$r,tier:"integration"}}')" code="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ -H "Authorization: Bearer $GH_TOKEN" \ -H "Accept: application/vnd.github+json" \ https://api.github.com/repos/id3as/norsk-ctl/dispatches -d "$body")" [ "$code" = "204" ] || { echo "::error::report to norsk-ctl failed (HTTP $code)"; exit 1; }