# One bump per repo per night (ci-workflows docs/dashboard/ci-fragility.md, # proposal B2). Keeps the product fresh against everything it builds on, in one # run and one PR: # 1. dev-kit: @norskvideo/ctl-dev-kit to npm `latest`, and every convention # copy re-synced from it (sync-drift) -- this workflow included; # 2. ctl-packages: the @norskvideo/ctl-* and moq-* family to npm `latest`; # 3. pins: the engine and Studio image pair from the nightly entry of # norsk-studio's channel manifest, and flake.nix's norsk-ctl pin to the # `latest` channel. # Then ONE lockfile refresh across every install root, the quick gate (drift, # lint, typecheck, unit), and ONE PR on bump/nightly, force-pushed, listing # every component that moved, from and to, and anything refused, and why. # # Why one, not three. upgrade-latest, sync-dev-kit and sync-ctl-packages each # edited package.json and bun.lock from main as it stood, and each was tested # alone. So whichever merged first left the others conflicting every night; # two bumps that needed each other could never pass alone (relay's harness PR # and its dev-kit PR, 2026-10-10: each was red on what the other fixed); and # main ended up with the combination of all three, which no PR had tested. # The trade-off: one bad component holds the night's good ones back. The PR # names what moved, so the culprit is visible, and the `components` input is # the escape hatch -- run it with the culprit left out to land the rest. # # Never pushes to main. bot-prs merges the PR once checks, smoke, integration # and the guides gate are green on its head, so a bad upstream release shows # up as a red bot PR and main stays green (proposal B). History, because each # step was a fix: the bumps first committed straight to main and let a RED # main be the alarm -- and nobody watches a product repo's own main: playout # sat red ~12h, then funke, then probe, inside three days (fleet review 04c, # class 8). Then they gated first and landed on green, which kept lint, # typecheck and unit failures off main but still let smoke, integration and # the guides go red there. Pinned for the class by # conventions/gate-before-main.test.ts. The branch is pushed with # CI_DISPATCH_TOKEN (a PAT): a GITHUB_TOKEN push would fire none of the PR's # workflows (Actions' recursion guard). # # Single-sourced in @norskvideo/ctl-dev-kit (conventions/bump.yml) and copied # verbatim into each product repo; the check:drift gate fails CI if a copy # diverges. Only the `product:` values below are per-repo (this repo's # dashboard key); everything else must match the dev-kit source. Never # hand-edit the copy -- edit the dev-kit source and re-sync. name: bump on: schedule: - cron: "17 4 * * *" # Sent, through ci-workflows' fan-out, when something this repo pins is # published (the norsk nightly, Studio's manifest pair, norsk-ctl latest), so # the pins move the same day rather than at the next 04:17. client_payload # may name components; it defaults to the pins. repository_dispatch: types: [upstream-published] workflow_dispatch: inputs: components: description: "What to move: all, or any of dev-kit, ctl-packages, pins (comma-separated). Leave a bad component out to land the rest." default: all base: description: "The branch to bump and open the PR against. main, unless trialling a change on a throwaway branch." default: main permissions: contents: write pull-requests: write concurrency: # One bump at a time per branch. Never cancelled: a run cut short between its # push and its PR leaves the branch and the PR disagreeing. group: bump-${{ inputs.base || 'main' }} cancel-in-progress: false jobs: bump: runs-on: x64 outputs: status: ${{ steps.night.outputs.status }} summary: ${{ steps.night.outputs.summary }} devkit_selected: ${{ steps.select.outputs.devkit }} devkit_status: ${{ steps.night.outputs.devkit_status }} devkit_detail: ${{ steps.night.outputs.devkit_detail }} steps: # Runs BEFORE checkout, and does two things: points the harness's temp base # out of the workspace for every later step in this job, and clears whatever # the old repo-local default left behind on this runner. The body carries the # why. It fails the job only when legacy litter survives -- which would take # the checkout down a step later anyway, but report nothing useful. - name: Route test-temp out of the workspace (pre-checkout) run: | set -uo pipefail # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the # HOST daemon has not seen before is created by dockerd, as ROOT, on the # host -- even when the launch passes --container-user (reproduced # 2026-09-04). The harness's default base is repo-local for an OrbStack # inotify constraint that binds macOS dev and nothing here, so every run # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx` # cannot remove. That took commentary's checks AND integration red on # 2026-09-03. Use the sanctioned override, per-run and outside the # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped. base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" mkdir -p "$base" echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV" # Yesterday's bases hold dockerd-created root dirs, so a non-root rm # EACCESes. They cannot wedge a checkout, but they do fill the disk. for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true done # LEGACY: a runner that ran the old default still carries a workspace # test-temp, and a checkout with clean:true dies on it. Reap the holders # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a # RESTARTING leaked container re-creates the path as root seconds after # any clean. Match on mount SOURCE, not a label: the holders are untracked # by construction, because a killed run labels nothing. tt="$GITHUB_WORKSPACE/test-temp" [ -d "$tt" ] || exit 0 stuck="" for c in $(docker ps -aq 2>/dev/null || true); do if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then stuck="$stuck $c" fi done if [ -n "$stuck" ]; then echo "reaping containers holding mounts under $tt:$stuck" docker rm -f $stuck || true fi # The DIRECTORY, not just its contents: an empty but root-owned base # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root # container unlink the leaf. docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \ sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true # Fail HERE if it survived. The checkout fails on it either way, but # reports only an EACCES rmdir with no clue what was holding the path. if [ -e "$tt" ]; then echo "::error::$tt survived the pre-checkout clean" ls -lan "$tt" || true exit 1 fi - uses: actions/checkout@v5 with: token: ${{ secrets.CI_DISPATCH_TOKEN }} ref: ${{ inputs.base || 'main' }} clean: true # The escape hatch. One bad component blocks the night's PR for the good # ones: re-run with the others named to land them, and hold the bad one # back until it is fixed upstream. Not split automatically on a red PR: # that is costly and complex, and the filter covers the real cases. - name: Choose the components to move id: select env: COMPONENTS: ${{ inputs.components || github.event.client_payload.components || (github.event_name == 'repository_dispatch' && 'pins' || 'all') }} run: | set -euo pipefail devkit=0; ctl=0; pins=0 for c in $(printf '%s' "${COMPONENTS:-all}" | tr ',' ' '); do case "$c" in all) devkit=1; ctl=1; pins=1 ;; dev-kit) devkit=1 ;; ctl-packages) ctl=1 ;; pins) pins=1 ;; *) echo "::error::unknown component '$c': name all, or any of dev-kit, ctl-packages, pins"; exit 1 ;; esac done [ $((devkit + ctl + pins)) -gt 0 ] || { echo "::error::no component named: name all, or any of dev-kit, ctl-packages, pins"; exit 1; } echo "moving: dev-kit=$devkit ctl-packages=$ctl pins=$pins" { echo "devkit=$devkit"; echo "ctl=$ctl"; echo "pins=$pins"; } >> "$GITHUB_OUTPUT" # DEV-KIT first: a new dev-kit can change the conventions and the gates # themselves, this workflow included. Every tracked package.json, not just # the root: a nested workspace can pin the dev-kit too, and a root-only # rewrite left it behind (playout's tests/ sat on 0.1.x after the root # moved to 0.2, ran a guide proxy that built the wrong Studio prefix, and # first-light failed every night from 2026-09-07). The copied conventions # are re-synced from the new dev-kit once it is installed, below. - name: Move the dev-kit pin id: devkit if: steps.select.outputs.devkit == '1' run: | set -euo pipefail reg="https://registry.npmjs.org" pkg="@norskvideo/ctl-dev-kit" refuse() { echo "::warning::dev-kit refused: $1" { echo "changed=0"; echo "summary="; echo "refused=$1"; } >> "$GITHUB_OUTPUT" exit 0 } # The runner has no npm; read the dist-tag off the registry with curl+jq. new="$(curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$reg/@norskvideo%2Fctl-dev-kit" | jq -r '."dist-tags".latest // empty' || true)" [ -n "$new" ] || refuse "could not resolve $pkg latest" echo "new=$new" >> "$GITHUB_OUTPUT" mapfile -t manifests < <(git ls-files '*package.json' ':!:**/node_modules/**') moved="" for m in "${manifests[@]}"; do old="$(grep -oE "\"$pkg\"[[:space:]]*:[[:space:]]*\"[^\"]*\"" "$m" | head -1 | sed -E 's/.*"([^"]*)"$/\1/' || true)" # Absent here, or a spec that names no registry version: a workspace # member, or a local tarball or checkout someone pinned on purpose # (a trial of an unreleased dev-kit) -- moving it would undo that. case "$old" in "" | workspace:* | file:* | link:*) continue ;; esac old_bare="${old#^}"; old_bare="${old_bare#~}" [ "$old_bare" != "$new" ] || continue # Surgical single-line rewrite so the file keeps its formatting # (a jq reserialize would reflow it and trip the formatter). sed -i -E "s|(\"$pkg\"[[:space:]]*:[[:space:]]*\")[^\"]*(\")|\1^$new\2|" "$m" echo "dev-kit: $m $old -> ^$new" moved="$moved $old_bare" done if [ -z "$moved" ]; then echo "every dev-kit pin is already on the newest dev-kit ($new)." { echo "changed=0"; echo "summary="; echo "refused="; } >> "$GITHUB_OUTPUT" exit 0 fi # `old` names the lowest pin moved. old_bare="$(printf '%s\n' $moved | sort -V | head -1)" { echo "changed=1" echo "old=$old_bare" echo "moved=- dev-kit: $old_bare -> $new" echo "summary=dev-kit \`$new\`" echo "refused=" } >> "$GITHUB_OUTPUT" # CTL-* and MOQ-*: the test harness that drives smoke and integration, the # SDK, the foundation helpers, the product-template schema, and the # @norskvideo fork of moq-dev/moq the dashboards play through. Nothing used # to move these, so the pins drifted silently: on 2026-08-30 studio carried # a dev-kit pin bumped that morning beside an SDK three releases behind, # and three repos ran three vintages of the same smoke runner. The moq-* # members pin each other at one version, so all nine move together, # including the ones a product gets only transitively. A minor bump of # these (0.1 -> 0.2) is a breaking API change, which is what the red-gate # draft PR is for. The dev-kit is deliberately absent: it has its own step. # # Every version is resolved before any pin is rewritten, so a refusal # never leaves the family split. - name: Move the ctl-* and moq-* pins id: ctl if: steps.select.outputs.ctl == '1' run: | set -euo pipefail reg="https://registry.npmjs.org" family="ctl-test-harness ctl-sdk ctl-foundation ctl-product-template-schema moq-signals moq-flate moq-pattern moq-net moq-json moq-loc moq-msf moq-hang moq-watch" refuse() { echo "::warning::ctl-packages refused: $1" { echo "changed=0"; echo "summary="; echo "refused=$1"; } >> "$GITHUB_OUTPUT" exit 0 } declare -A latest=() for short in $family; do v="$(curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$reg/@norskvideo%2F$short" | jq -r '."dist-tags".latest // empty' || true)" [ -n "$v" ] || refuse "could not resolve @norskvideo/$short latest" latest[$short]="$v" done # Every tracked package.json, not just the root: a repo can pin the # same package in a nested workspace (playout pins the harness twice), # and a root-only rewrite would split the line across the two. mapfile -t manifests < <(git ls-files '*package.json' ':!:**/node_modules/**') moved=() parts=() moq="" for short in $family; do pkg="@norskvideo/$short" new="${latest[$short]}" hit=0 for m in "${manifests[@]}"; do old="$(grep -oE "\"$pkg\"[[:space:]]*:[[:space:]]*\"[^\"]*\"" "$m" | head -1 | sed -E 's/.*"([^"]*)"$/\1/' || true)" # Absent, or no registry version (workspace, file: or link:). case "$old" in "" | workspace:* | file:* | link:*) continue ;; esac old_bare="${old#^}"; old_bare="${old_bare#~}" [ "$old_bare" != "$new" ] || continue sed -i -E "s|(\"$pkg\"[[:space:]]*:[[:space:]]*\")[^\"]*(\")|\1^$new\2|" "$m" echo "$m: $pkg $old -> ^$new" moved+=("- $pkg: $old -> ^$new ($m)") hit=1 done [ "$hit" = 1 ] || continue # One summary entry per package, and one for the whole moq-* line # when its members agree, which they do by construction. case "$short" in moq-*) case " $moq " in *" $new "*) ;; *) moq="$moq $new" ;; esac ;; *) parts+=("$short \`$new\`") ;; esac done for v in $moq; do parts+=("moq \`$v\`"); done if [ ${#moved[@]} -eq 0 ]; then echo "every ctl-* and moq-* pin is already on its newest published version." { echo "changed=0"; echo "summary="; echo "refused="; } >> "$GITHUB_OUTPUT" exit 0 fi summary="$(printf '%s, ' "${parts[@]}")" { echo "changed=1" echo "summary=${summary%, }" echo "refused=" echo "moved<> "$GITHUB_OUTPUT" # PINS last: the {engine, studio image} pair from the nightly entry of # norsk-studio's channel manifest -- the pair studio TESTED together -- # never from the two channels separately, plus flake.nix's norsk-ctl pin # (ctlVersion + the per-platform hashes) to the `latest` channel. That pin # is not dev-only: every CI tier runs the norsk-ctl it names, so a # product's result belongs to a commit (ctl-pinned-in-repo.test.ts). # Version discovery anchors on manifest.seed.json (the machine-writable # source of truth for the launched {media, studio} image pair); npm pins on # the same version string (media build libraries, the # @norskvideo/norsk-studio* family in whichever tracked package.json # depends on them) move in lockstep via a global string-replace of old->new # across every tracked file -- which also updates the version-pin guard # test -- except docs/internal/, whose records name the version something # happened on and must keep saying so (#642). A repo with no studio npm # deps needs no `overrides` block at all. # # A REFUSAL (the channels disagree, a pin is stranded, there is no # ctlVersion, a channel cannot be read) refuses the pins alone: the step # says why and moves nothing, and the night's other components still # reach their PR. Everything is resolved and checked BEFORE anything is # rewritten, so a refusal never leaves the engine moved and studio not. - name: Move the engine and Studio pair and the norsk-ctl pin id: pins if: steps.select.outputs.pins == '1' run: | set -euo pipefail reg="https://registry.npmjs.org" changed=0 refuse() { echo "::warning::pins refused: $1" { echo "changed=0"; echo "summary="; echo "refused=$1"; } >> "$GITHUB_OUTPUT" exit 0 } # NORSK-CTL: the version flake.nix's ctlVersion names is the daemon every # CI tier runs as well as the one the dev shell launches, so it moves here # like the engine does and the bump PR's tiers are what test it # (ci-workflows docs/dashboard/ci-fragility.md, proposal A). ctl_base="https://s3.eu-west-1.amazonaws.com/norsk.video/norsk-ctl" old_ctl="$( { grep -oE 'ctlVersion = "[^"]+"' flake.nix 2>/dev/null || true; } | head -1 | sed -E 's/.*"(.*)"/\1/')" [ -n "$old_ctl" ] || refuse "flake.nix pins no ctlVersion. CI runs the norsk-ctl it names, so add ctlVersion = \"\"; to flake.nix." new_ctl="$(curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$ctl_base/latest" || true)" [ -n "$new_ctl" ] || refuse "could not resolve norsk-ctl's latest channel" # The per-platform hash of each platform the flake fetches the daemon # for (a Studio product's flake fetches all four; an sdk-app's none). # Fetched now, not after the engine has moved: a missing sidecar used # to leave the repo half-bumped. Mirrors build/refresh-ctl-pin.sh. declare -A ctl_sri=() if [ "$old_ctl" != "$new_ctl" ]; then for p in linux-x64 linux-arm64 darwin-arm64 darwin-x64; do grep -qF "plat = \"$p\";" flake.nix || continue hex="$(curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$ctl_base/$new_ctl/norsk-ctl-$new_ctl-$p.sha256" | awk '{print $1}' || true)" [ -n "$hex" ] || refuse "norsk-ctl $new_ctl has no sha256 sidecar for $p" ctl_sri[$p]="$(nix hash convert --hash-algo sha256 --to sri "$hex")" done fi # Only the components that actually moved get an entry, so a no-op # night posts nothing. parts=() moved=() # ONE TESTED PAIR, NOT TWO CHANNELS. Resolving the engine off # norsk-sdk's nightly dist-tag and the image off Docker Hub's nightly # tag produced pairs nobody had tested whenever one channel moved # without the other. On 2026-10-04 studio's nightly had been held by # its test gate since 09-28 while the engine walked on to 10-02, a # breaking SDK change landed in between, and commentary's audio mix # produced nothing on the old-studio / new-engine pair (norsk task 112). # # Consequence, deliberately: when studio's nightly stalls, the engine # stalls with it (or steps BACK to the pair), rather than running ahead # untested. Unblocking studio's nightly is how products move forward. manifest_url="https://s3.eu-west-1.amazonaws.com/norsk.video/channel-manifest.json" pair="$(curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$manifest_url" | jq -c '.channels.nightly // empty' || true)" [ -n "$pair" ] || refuse "no nightly pair in $manifest_url" tag_re='^[0-9]+\.[0-9]+\.[0-9]+-[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9a-f]+$' new_media="$(jq -r '(.norsk // "") | split(":") | .[1] // ""' <<<"$pair")" new_studio_img="$(jq -r '(.studio // "") | split(":") | .[1] // ""' <<<"$pair")" printf '%s' "$new_media" | grep -qE "$tag_re" || refuse "the manifest's nightly norsk image is not a dated nightly tag: $pair" printf '%s' "$new_studio_img" | grep -qE "$tag_re" || refuse "the manifest's nightly studio image is not a dated nightly tag: $pair" echo "nightly pair: norsk $new_media + studio $new_studio_img" # MEDIA: the npm packages and the launched image are built from the SAME # commit (their hash suffixes match), so the pair's engine version drives # the whole family: overrides, sub-package deps, the manifest image ref, # the version-pin test -- every occurrence EXCEPT the lockfile, which bun # install regenerates. old_media="$(jq -r '.latest.media | split(":")[1]' manifest.seed.json)" # The rewrite below is a LITERAL replace of the seed value, so it only # ever touches pins that already equal it. A pin that has drifted off # it is invisible to this bump and to every later one, permanently: # funke launched a 2026-09-30 engine against 2026-09-05 npm pins for a # month that way. Not auto-rewritten -- a pin held back on purpose is # someone's decision -- so refuse, name them, and let a human fix the # pin or the intent. stranded="$(git grep -hoE '"@norskvideo/norsk-(api|sdk|auto-manager|manager-sdk)"[[:space:]]*:[[:space:]]*"[0-9]+\.[0-9]+\.[0-9]+-[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9a-f]+"' -- '*package.json' \ | sed -E 's/.*"([^"]+)"$/\1/' | sort -u | grep -vxF "$old_media" | grep -vxF "$new_media" || true)" if [ -n "$stranded" ]; then refuse "npm norsk pins are stranded off the nightly channel: $(printf '%s' "$stranded" | tr '\n' ' ')- the media family (api, sdk, auto-manager, manager-sdk) and the launched image are built from one commit, so every pin must read $new_media, and no later bump can reach these. Set them to $new_media." fi # STUDIO: the studio npm packages and the studio image are built from # DIFFERENT commits -- their hash suffixes never match -- so the image # comes from the pair and the build libraries from npm's nightly. The # libraries must belong to the same pair: their norsk-sdk dependency has # to name the pair's engine, and their version line the image's. A # mismatch means the channels are inconsistent; refuse rather than pin # a combination nobody tested. (Studio's publish installs the SDK it # builds against as a DEV dependency, so that is where the pairing lives.) # # The old npm pin is discovered from the repo's REAL studio deps: any # exact nightly on a @norskvideo/norsk-studio* package in a tracked # package.json. A repo with none skips the lib entirely. Distinct pins # disagreeing across files refuse rather than half-rewrite. old_studio_lib="$(git grep -hoE '"@norskvideo/norsk-studio[^"]*"[[:space:]]*:[[:space:]]*"[0-9]+\.[0-9]+\.[0-9]+-[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9a-f]+"' -- '*package.json' \ | sed -E 's/.*"([^"]+)"$/\1/' | sort -u || true)" if [ "$(printf '%s' "$old_studio_lib" | grep -c .)" -gt 1 ]; then refuse "studio npm pins disagree across package.json files: $(printf '%s' "$old_studio_lib" | tr '\n' ' ')" fi old_studio_img="$(jq -r '.latest.studio | split(":")[1]' manifest.seed.json)" new_studio_lib="" if [ -n "$old_studio_lib" ]; then studio_doc="$(curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$reg/@norskvideo%2Fnorsk-studio" || true)" new_studio_lib="$(jq -r '."dist-tags".nightly // empty' <<<"$studio_doc" 2>/dev/null || true)" [ -n "$new_studio_lib" ] || refuse "could not resolve norsk-studio npm nightly" lib_sdk="$(jq -r --arg v "$new_studio_lib" \ '.versions[$v] | ((.devDependencies // {}) + (.peerDependencies // {}) + (.dependencies // {}))["@norskvideo/norsk-sdk"] // empty' <<<"$studio_doc")" lib_sdk="${lib_sdk#^}"; lib_sdk="${lib_sdk#~}" if [ "$lib_sdk" != "$new_media" ]; then refuse "norsk-studio npm nightly $new_studio_lib was built against norsk-sdk ${lib_sdk:-}, not the tested pair's engine $new_media. The npm channel and the channel manifest disagree; not pinning a combination nobody tested." fi if [ "${new_studio_lib%%-*}" != "${new_studio_img%%-*}" ]; then refuse "norsk-studio npm nightly $new_studio_lib is on a different version line from the pair's image $new_studio_img." fi fi # Everything is resolved and checked; from here on, rewrite. echo "media: $old_media -> $new_media" echo "studio lib: ${old_studio_lib:-} -> ${new_studio_lib:-}" echo "studio image: $old_studio_img -> $new_studio_img" if [ "$old_media" != "$new_media" ]; then git grep -lF "$old_media" -- . ':!bun.lock' ':!docs/internal' | xargs -r sed -i "s|$old_media|$new_media|g" changed=1 parts+=("media \`$new_media\`") moved+=("- media: $old_media -> $new_media") fi # Rewrite the image ref FIRST, then the bare library version -- so even # when the two coincide, the image ref has already moved and the # bare-version rewrite only touches the npm pins. if [ "$old_studio_img" != "$new_studio_img" ]; then git grep -lF "norskvideo/norsk-studio:$old_studio_img" -- . ':!bun.lock' ':!docs/internal' \ | xargs -r sed -i "s|norskvideo/norsk-studio:$old_studio_img|norskvideo/norsk-studio:$new_studio_img|g" # And the tag on its own: a product can carry it unprefixed (onair2026's # committed template input has `"studioTag": ""`), which the # prefixed rewrite never reached (2026-10-04). Safe because the image # tag and the npm library version are different strings; if they ever # coincide, the lib rewrite below owns that string. if [ "$old_studio_img" != "$old_studio_lib" ]; then # `|| true`: after the prefixed rewrite most repos have no bare # occurrence left, and git grep exits 1 on no match (pipefail). { git grep -lF "$old_studio_img" -- . ':!bun.lock' ':!docs/internal' || true; } | xargs -r sed -i "s|$old_studio_img|$new_studio_img|g" fi changed=1 moved+=("- studio: $old_studio_img -> $new_studio_img") fi if [ -n "$new_studio_lib" ] && [ "$old_studio_lib" != "$new_studio_lib" ]; then git grep -lF "$old_studio_lib" -- . ':!bun.lock' ':!docs/internal' | xargs -r sed -i "s|$old_studio_lib|$new_studio_lib|g" changed=1 moved+=("- studio-lib: $old_studio_lib -> $new_studio_lib") fi # One studio summary entry, not two -- the launched image tag is the # runtime-relevant one; the lib tag only if the image held but the # library advanced. if [ "$old_studio_img" != "$new_studio_img" ]; then parts+=("studio \`$new_studio_img\`") elif [ -n "$new_studio_lib" ] && [ "$old_studio_lib" != "$new_studio_lib" ]; then parts+=("studio-lib \`$new_studio_lib\`") fi # --- norsk-ctl pin (flake.nix) ------------------------------------ if [ "$old_ctl" != "$new_ctl" ]; then for p in "${!ctl_sri[@]}"; do sri="${ctl_sri[$p]}" perl -0pi -e "s{(plat = \"$p\";\\s*hash = \")[^\"]*(\")}{\${1}$sri\${2}}" flake.nix done perl -0pi -e "s{(ctlVersion = \")[^\"]*(\")}{\${1}$new_ctl\${2}}" flake.nix echo "ctl: $old_ctl -> $new_ctl" changed=1 parts+=("norsk-ctl \`$new_ctl\`") moved+=("- norsk-ctl: $old_ctl -> $new_ctl") fi summary="" if [ ${#parts[@]} -gt 0 ]; then summary="$(printf '%s, ' "${parts[@]}")" summary="${summary%, }" fi { echo "changed=$changed" echo "summary=$summary" echo "refused=" echo "moved<> "$GITHUB_OUTPUT" # The night's one answer, from the three components': did anything move, # what (one line per pin, from and to, for the PR body and the brief), # what was refused and why, and what the dashboard's nightly and dev-kit # cells say. Through env, never pasted into the script: the summaries wrap # versions in backticks, and inside a script those run as commands. - name: Gather what the night moved id: night env: DEVKIT_SELECTED: ${{ steps.select.outputs.devkit }} DEVKIT_CHANGED: ${{ steps.devkit.outputs.changed }} DEVKIT_OLD: ${{ steps.devkit.outputs.old }} DEVKIT_NEW: ${{ steps.devkit.outputs.new }} DEVKIT_MOVED: ${{ steps.devkit.outputs.moved }} DEVKIT_SUMMARY: ${{ steps.devkit.outputs.summary }} DEVKIT_REFUSED: ${{ steps.devkit.outputs.refused }} CTL_SELECTED: ${{ steps.select.outputs.ctl }} CTL_CHANGED: ${{ steps.ctl.outputs.changed }} CTL_MOVED: ${{ steps.ctl.outputs.moved }} CTL_SUMMARY: ${{ steps.ctl.outputs.summary }} CTL_REFUSED: ${{ steps.ctl.outputs.refused }} PINS_SELECTED: ${{ steps.select.outputs.pins }} PINS_CHANGED: ${{ steps.pins.outputs.changed }} PINS_MOVED: ${{ steps.pins.outputs.moved }} PINS_SUMMARY: ${{ steps.pins.outputs.summary }} PINS_REFUSED: ${{ steps.pins.outputs.refused }} run: | set -euo pipefail changed=0; names=(); moved=(); parts=(); refused=(); why=() take() { local name="$1" selected="$2" did="$3" lines="$4" summary="$5" refusal="$6" [ "$selected" = 1 ] || return 0 if [ -n "$refusal" ]; then refused+=("- $name: $refusal"); why+=("$name ($refusal)") fi [ "$did" = 1 ] || return 0 changed=1; names+=("$name") [ -z "$lines" ] || moved+=("$lines") [ -z "$summary" ] || parts+=("$summary") } take dev-kit "${DEVKIT_SELECTED:-0}" "${DEVKIT_CHANGED:-0}" "${DEVKIT_MOVED:-}" "${DEVKIT_SUMMARY:-}" "${DEVKIT_REFUSED:-}" take ctl-packages "${CTL_SELECTED:-0}" "${CTL_CHANGED:-0}" "${CTL_MOVED:-}" "${CTL_SUMMARY:-}" "${CTL_REFUSED:-}" take pins "${PINS_SELECTED:-0}" "${PINS_CHANGED:-0}" "${PINS_MOVED:-}" "${PINS_SUMMARY:-}" "${PINS_REFUSED:-}" join() { local sep="$1" out; shift; out="$(printf "%s$sep" "$@")"; printf '%s' "${out%"$sep"}"; } summary="$(join ', ' ${parts[@]+"${parts[@]}"})" if [ ${#why[@]} -gt 0 ]; then summary="${summary:+$summary; }refused: $(join ', ' "${why[@]}")" status=failure else status=success fi if [ -n "${DEVKIT_REFUSED:-}" ]; then devkit_status=failure; devkit_detail="refused: $DEVKIT_REFUSED" elif [ "${DEVKIT_CHANGED:-0}" = 1 ]; then devkit_status=success; devkit_detail="$DEVKIT_OLD -> $DEVKIT_NEW" else devkit_status=success; devkit_detail="current ${DEVKIT_NEW:-}" fi { echo "changed=$changed" echo "components=$(join ', ' ${names[@]+"${names[@]}"})" echo "summary=$summary" echo "status=$status" echo "devkit_status=$devkit_status" echo "devkit_detail=$devkit_detail" echo "moved<> "$GITHUB_OUTPUT" # ONE refresh, after every pin has moved: three bots each refreshing from # main as it stood is what made their PRs conflict on bun.lock every # night. NOT continue-on-error, unlike the gate below: a refresh that dies # part-way (the runner OOM-killed one on 2026-10-03) leaves bumped # manifests beside the old lockfiles, and carrying on committed exactly # that. Fail here instead, before anything is committed. - name: Refresh every lockfile against the new pins if: steps.night.outputs.changed == '1' env: DEVKIT_CHANGED: ${{ steps.devkit.outputs.changed }} CTL_CHANGED: ${{ steps.ctl.outputs.changed }} run: | nix develop .#build --command bash -c ' set -euo pipefail # --ignore-scripts: a repo whose own prepare (relay: tsc) fails on # main would kill the bump that fixes it (2026-10-14). The gate # below installs for real, so a tree that does not build drafts. # Per INSTALL ROOT, not per repo: the rewrites walk every tracked # package.json, and a manifest outside the root workspace list (funke # proxy/ui) owns a lockfile a root-only install never regenerates. for lock in $(git ls-files "*bun.lock"); do (cd "$(dirname "$lock")" && bun install --ignore-scripts) # bun install keeps a locked version its range still admits, so a # nested workspace could stay on an old dev-kit under a new pin. # Only where the lockfile already holds it: bun update ADDS a # package it does not find, and put the dev-kit into funke proxy/ui. if [ "$DEVKIT_CHANGED" = 1 ] && grep -q "\"@norskvideo/ctl-dev-kit@" "$lock"; then (cd "$(dirname "$lock")" && bun update --ignore-scripts @norskvideo/ctl-dev-kit) fi done # A ctl-* package also arrives TRANSITIVELY, and those never move on # their own: probe sat on ctl-foundation 0.1.14 from 2026-09-02 while # 0.1.19 was current, through ctl-sdk and the schema, and the gate # went red on an export the stale copy lacked. Same family as the ctl # step, named again because the two are separate shells. Not every # install root has every member; bun says so and this moves on. if [ "$CTL_CHANGED" = 1 ]; then for pkg in ctl-test-harness ctl-sdk ctl-foundation ctl-product-template-schema moq-signals moq-flate moq-pattern moq-net moq-json moq-loc moq-msf moq-hang moq-watch; do for lock in $(git ls-files "*bun.lock"); do (cd "$(dirname "$lock")" && bun update --ignore-scripts "@norskvideo/$pkg") || true done done fi ' # sync-drift re-derives each convention copy from the NEW dev-kit's # canonical, keeping the per-repo bits the gate masks; it is how a # convention change (this workflow included) reaches the product. Then # the quick gate. continue-on-error, so a red gate still opens the PR -- # which is why the steps after it read `outcome`, not `conclusion`. - name: Re-sync the conventions and run the product gate id: gate if: steps.night.outputs.changed == '1' continue-on-error: true env: DEVKIT_CHANGED: ${{ steps.devkit.outputs.changed }} run: | nix develop .#build --command bash -c ' set -euo pipefail if [ "$DEVKIT_CHANGED" = 1 ]; then bun run node_modules/@norskvideo/ctl-dev-kit/conventions/sync-drift.ts fi bun run check:drift bun run lint bun run typecheck bun run test:unit ' # The night's only way to main is this PR. bot-prs merges it once every # tier that runs on a bot PR -- checks, smoke, integration and # build-image's guides gate, wherever this repo has them -- is green on # its head (ci-workflows docs/dashboard/ci-fragility.md, proposals B and # B2). A red quick gate opens it as a draft, and a later green run makes # it ready again (draft is not settable by PATCH, so through GraphQL). - name: Open (or update) the bump PR if: steps.night.outputs.changed == '1' env: API_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }} GATE: ${{ steps.gate.outcome }} BASE: ${{ inputs.base || 'main' }} # Through env, never pasted into the script: the summary wraps each # version in backticks, and inside a double-quoted shell string those # ran as commands ("1.3.1-...: command not found"). MOVED: ${{ steps.night.outputs.moved }} REFUSED: ${{ steps.night.outputs.refused }} SUMMARY: ${{ steps.night.outputs.summary }} COMPONENTS: ${{ steps.night.outputs.components }} DEVKIT_SELECTED: ${{ steps.select.outputs.devkit }} CTL_SELECTED: ${{ steps.select.outputs.ctl }} PINS_SELECTED: ${{ steps.select.outputs.pins }} run: | set -euo pipefail branch="bump/nightly" title="Nightly bump: $COMPONENTS" repo="${{ github.repository }}"; owner="${repo%%/*}" api="https://api.github.com/repos/$repo/pulls" # This PR supersedes any older bump PR on the same base, and the old # bots' PRs for every component this run covered: upgrade-latest's # (pins), sync-dev-kit's (dev-kit) and sync-ctl-packages' (ctl-*). # Left open they would sit red forever, with no bot left to rebuild # them. A component the filter left out keeps its old PR. Only PRs on # this run's base: a trial against a throwaway branch touches nothing # on main. Best-effort: a failure here must not fail the bump. families=("bump/") [ "$DEVKIT_SELECTED" != 1 ] || families+=("sync-dev-kit/") [ "$CTL_SELECTED" != 1 ] || families+=("chore/sync-ctl-packages") [ "$PINS_SELECTED" != 1 ] || families+=("upgrade-latest/") close_superseded() { local keep="$1" n fam fam="$(printf '%s\n' "${families[@]}" | jq -R . | jq -sc .)" auth=(-H "Authorization: Bearer $API_TOKEN" -H "Accept: application/vnd.github+json") for n in $(curl -sS "${auth[@]}" "$api?state=open&per_page=100" \ | jq -r --arg repo "$repo" --arg base "$BASE" --argjson fam "$fam" --argjson keep "$keep" ' .[] | select(.head.repo.full_name == $repo and .base.ref == $base and .number != $keep) | .head.ref as $ref | select(any($fam[]; . as $prefix | if ($prefix | endswith("/")) then ($ref | startswith($prefix)) else $ref == $prefix end)) | .number'); do curl -sS -o /dev/null "${auth[@]}" -X POST "https://api.github.com/repos/$repo/issues/$n/comments" \ -d "$(jq -nc --arg b "Superseded by #$keep, the nightly bump (bump.yml), which moves this component now." '{body:$b}')" curl -sS -o /dev/null "${auth[@]}" -X PATCH "$api/$n" -d '{"state":"closed"}' echo "closed superseded PR #$n" done } git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git checkout -B "$branch" git add -A git commit -q -F - </dev/null && [ "$(git rev-parse 'HEAD^{tree}')" = "$(git rev-parse 'FETCH_HEAD^{tree}')" ]; then echo "$branch already holds this bump; not re-pushing it" else git push -f origin "$branch" fi if [ "$GATE" = "success" ]; then verdict="Gate green: check:drift, lint, typecheck and unit tests pass on the bumped pins." draft=false else verdict="**Gate red**: check:drift, lint, typecheck or unit tests fail on the bumped pins, so this is a draft. Run the gate on this branch to see which (\`bun install\`, \`bun run check:drift\`, \`bun run lint && bun run typecheck && bun run test:unit\`). Fix it here, or re-run bump.yml with the culprit left out of \`components\` to land the rest." draft=true fi lines=( "The nightly bump (bump.yml): one PR for every component that moved, tested together." "" "Moved, from and to:" "" "" "$MOVED" "" ) if [ -n "$REFUSED" ]; then lines+=( "" "Refused, and left where they were (the run's log says more):" "" "" "$REFUSED" "" ) fi lines+=( "" "$verdict" "" "bot-prs merges this once checks, smoke, integration and the guides gate" "(build-image) have all passed on its head, wherever this repo runs them." "Until then $BASE does not move. One bad component holds the night back: to land" "the rest, run \`gh workflow run bump.yml -f components=\`." ) body="$(printf '%s\n' "${lines[@]}")" # gh is not on the self-hosted runner PATH (it lives only inside # `nix develop`, and this step runs outside it), so create/update the PR # straight against the REST API with the same PAT that pushed the branch. # The PAT (not GITHUB_TOKEN) is what lets the PR's own workflows run, # which is the whole point: the bump is judged before it lands. curl_api=(--fail-with-body -sS -H "Authorization: Bearer $API_TOKEN" -H "Accept: application/vnd.github+json") standing="$(curl "${curl_api[@]}" "$api?head=$owner:$branch&base=$BASE&state=open")" existing="$(jq -r '.[0].number // empty' <<<"$standing")" if [ -n "$existing" ]; then curl "${curl_api[@]}" -X PATCH "$api/$existing" \ --data "$(jq -nc --arg t "$title" --arg b "$body" '{title:$t, body:$b}')" > /dev/null number="$existing" echo "updated PR #$number" is_draft="$(jq -r '.[0].draft // false' <<<"$standing")" node="$(jq -r '.[0].node_id // empty' <<<"$standing")" mutation="" if [ "$draft" = true ] && [ "$is_draft" != true ]; then mutation=convertPullRequestToDraft; fi if [ "$draft" = false ] && [ "$is_draft" = true ]; then mutation=markPullRequestReadyForReview; fi if [ -n "$mutation" ] && [ -n "$node" ]; then curl "${curl_api[@]}" -X POST "https://api.github.com/graphql" \ --data "$(jq -nc --arg m "$mutation" --arg id "$node" '{query: ("mutation($id: ID!) { " + $m + "(input: {pullRequestId: $id}) { pullRequest { isDraft } } }"), variables: {id: $id}}')" > /dev/null || echo "::warning::could not $mutation #$number" echo "$mutation #$number" fi else number="$(curl "${curl_api[@]}" -X POST "$api" \ --data "$(jq -nc --arg t "$title" --arg b "$body" --arg h "$branch" --arg base "$BASE" --argjson d "$draft" \ '{title:$t, body:$b, head:$h, base:$base, draft:$d}')" | jq -r .number)" echo "opened PR #$number" fi close_superseded "$number" || true - name: Report "nothing to bump" if: steps.night.outputs.changed != '1' run: echo "Every selected component is already current; nothing to bump." - name: Reap the containers this job launched # Whatever the tests did. A teardown cut short by a test timeout leaves # its instance running on the shared Docker daemon (see JOB_REAP_STEP in # conventions/workflow-shell.ts); everything this job launched mounts its # own test-temp, so that is what goes. if: always() run: | base="${NORSK_CTL_TEST_TMP:-}" tt="$GITHUB_WORKSPACE/test-temp" s="" for c in $(docker ps -aq); do src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi done if [ -n "$s" ]; then echo "::warning::removing containers this job left running:$s" docker rm -f $s fi # Report to the aggregated product CI dashboard (id3as/ci-workflows): the # night as the `nightly` pipeline, and the dev-kit component as the `dev-kit` # one -- the board keeps that column, fed by this run. !cancelled() so a # failed bump shows a sad pony while a cancelled run does not false-red the # board; a run against any branch but main is a trial, and reports nothing. # A refused component reports red although its PR opened: someone has to # look at why. notify: needs: bump if: ${{ !cancelled() && (inputs.base || 'main') == 'main' }} runs-on: x64 steps: # Runs BEFORE checkout, and does two things: points the harness's temp base # out of the workspace for every later step in this job, and clears whatever # the old repo-local default left behind on this runner. The body carries the # why. It fails the job only when legacy litter survives -- which would take # the checkout down a step later anyway, but report nothing useful. - name: Route test-temp out of the workspace (pre-checkout) run: | set -uo pipefail # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the # HOST daemon has not seen before is created by dockerd, as ROOT, on the # host -- even when the launch passes --container-user (reproduced # 2026-09-04). The harness's default base is repo-local for an OrbStack # inotify constraint that binds macOS dev and nothing here, so every run # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx` # cannot remove. That took commentary's checks AND integration red on # 2026-09-03. Use the sanctioned override, per-run and outside the # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped. base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" mkdir -p "$base" echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV" # Yesterday's bases hold dockerd-created root dirs, so a non-root rm # EACCESes. They cannot wedge a checkout, but they do fill the disk. for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true done # LEGACY: a runner that ran the old default still carries a workspace # test-temp, and a checkout with clean:true dies on it. Reap the holders # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a # RESTARTING leaked container re-creates the path as root seconds after # any clean. Match on mount SOURCE, not a label: the holders are untracked # by construction, because a killed run labels nothing. tt="$GITHUB_WORKSPACE/test-temp" [ -d "$tt" ] || exit 0 stuck="" for c in $(docker ps -aq 2>/dev/null || true); do if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then stuck="$stuck $c" fi done if [ -n "$stuck" ]; then echo "reaping containers holding mounts under $tt:$stuck" docker rm -f $stuck || true fi # The DIRECTORY, not just its contents: an empty but root-owned base # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root # container unlink the leaf. docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \ sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true # Fail HERE if it survived. The checkout fails on it either way, but # reports only an EACCES rmdir with no clue what was holding the path. if [ -e "$tt" ]; then echo "::error::$tt survived the pre-checkout clean" ls -lan "$tt" || true exit 1 fi - uses: actions/checkout@v5 with: clean: true - id: meta env: NIGHT: ${{ needs.bump.outputs.status }} DEVKIT: ${{ needs.bump.outputs.devkit_status }} run: | if [ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" = "true" ]; then echo "state=failure" >> "$GITHUB_OUTPUT" echo "devkit=failure" >> "$GITHUB_OUTPUT" else echo "state=${NIGHT:-success}" >> "$GITHUB_OUTPUT" echo "devkit=${DEVKIT:-success}" >> "$GITHUB_OUTPUT" fi - uses: ./.github/actions/ci-status-dispatch with: token: ${{ secrets.CI_DISPATCH_TOKEN }} product: __PRODUCT__ pipeline: nightly status: ${{ steps.meta.outputs.state }} detail: ${{ needs.bump.outputs.summary }} - if: needs.bump.outputs.devkit_selected == '1' uses: ./.github/actions/ci-status-dispatch with: token: ${{ secrets.CI_DISPATCH_TOKEN }} product: __PRODUCT__ pipeline: dev-kit status: ${{ steps.meta.outputs.devkit }} detail: ${{ needs.bump.outputs.devkit_detail }} - name: Reap the containers this job launched # Whatever the tests did. A teardown cut short by a test timeout leaves # its instance running on the shared Docker daemon (see JOB_REAP_STEP in # conventions/workflow-shell.ts); everything this job launched mounts its # own test-temp, so that is what goes. if: always() run: | base="${NORSK_CTL_TEST_TMP:-}" tt="$GITHUB_WORKSPACE/test-temp" s="" for c in $(docker ps -aq); do src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi done if [ -n "$s" ]; then echo "::warning::removing containers this job left running:$s" docker rm -f $s fi