# Reference: three-tier E2E gating workflow (devaudit#152 follow-up, v0.1.53) # # Copy this into your consumer-owned .github/workflows/e2e-regression.yml # to adopt the 3-tier model: smoke (every develop push, fast) / critical # (consumer-enabled release PR, ~10-15 min target) / regression after a # successful production deployment status (or manual dispatch). # # The framework does NOT sync this file automatically — your consumer # owns its e2e-regression.yml. Apply the patterns below to your own # file; keep any consumer-specific env / matrix / runner customisations. # # Tier definitions: # - smoke — runs on develop push via ci.yml (no change here) # - critical — Playwright project that selects e2e/smoke/ + e2e/critical/ # - regression — Playwright project that selects all e2e/**/*.spec.ts # # devaudit-installer#787 — this workflow's own `name:`/job name stay # "E2E Regression" for both tiers (renaming risks breaking a consumer's # existing branch-protection required-check-name reference), but the # portal-facing check label compliance-evidence.yml.template builds IS # tier-aware: "E2E Critical (pre-merge)" vs "E2E Regression (post-deploy # production)". See docs/e2e-test-tiers.md. # # playwright.config.ts must define the `critical` project for this to # fire; if it doesn't, the gate falls back to the existing `smoke` # project so PR-to-main stays green during migration. # # A post-merge regression is release evidence, not a best-effort background # task. Keep its timeout below the job timeout so the workflow can retain # partial evidence and report a terminal timeout outcome to DevAudit. name: E2E Regression on: pull_request: branches: [main] # critical-tier gate before merge deployment_status: types: [created] # full regression only after successful production deploy workflow_dispatch: inputs: specs: description: 'Optional: space-separated spec paths or --grep pattern for a scoped run. Empty = full regression.' required: false runner_label: description: 'Optional one-run runner label override. Blank uses repository CI_RUNNER_LABEL.' required: false permissions: contents: read issues: write # post-merge auto-issue on regression failure concurrency: group: e2e-regression-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: e2e: name: E2E Regression Tests if: >- github.event_name != 'deployment_status' || (github.event.deployment_status.state == 'success' && (github.event.deployment.environment == 'production' || github.event.deployment.environment == 'prod' || endsWith(github.event.deployment.environment, '/ production') || endsWith(github.event.deployment.environment, '/production'))) # Honors the same CI_RUNNER_LABEL repository variable every devaudit- # generated workflow reads (ci.yml, feature-e2e.yml, ...) — falls back to # 'github-ci' -> 'ubuntu-latest' when unset, so this is a no-op for a # consumer with no self-hosted runner configured. runs-on: ${{ (inputs.runner_label || vars.CI_RUNNER_LABEL || 'github-ci') == 'github-ci' && 'ubuntu-latest' || (inputs.runner_label || vars.CI_RUNNER_LABEL || 'github-ci') }} # The full regression target is about 35 minutes. Leave time to archive # partial output before GitHub terminates the job. timeout-minutes: 55 steps: - uses: actions/checkout@v6 with: fetch-depth: 0 # for E2E_NEW_SPECS computation ref: ${{ github.event.deployment.sha || github.sha }} - uses: actions/setup-node@v6 with: node-version: '22' # match your project cache: 'npm' - name: Install dependencies run: npm ci --legacy-peer-deps - name: Install Playwright browsers run: npx playwright install --with-deps chromium # Decide which Playwright project to run based on the trigger. # PR-to-main uses critical with smoke fall-back; a consumer-enabled # post-merge push runs the full regression project; workflow_dispatch # accepts an optional spec filter. - name: Determine E2E project + spec selector id: select run: | set -euo pipefail EVENT="${{ github.event_name }}" case "$EVENT" in pull_request) if grep -qE "name:\s*['\"]critical['\"]" playwright.config.ts 2>/dev/null; then echo "project=critical" >> "$GITHUB_OUTPUT" echo "Using critical-tier project (smoke + e2e/critical/)" else echo "project=smoke" >> "$GITHUB_OUTPUT" echo "::warning::No 'critical' Playwright project defined; falling back to smoke. See e2e-test-engineer/references/e2e-regression-3-tier.yml + the Phase 3 tier-classification guide." fi echo "specs=" >> "$GITHUB_OUTPUT" ;; deployment_status|schedule) echo "project=regression" >> "$GITHUB_OUTPUT" echo "specs=" >> "$GITHUB_OUTPUT" echo "Running full regression project" ;; workflow_dispatch) echo "project=regression" >> "$GITHUB_OUTPUT" echo "specs=${{ github.event.inputs.specs }}" >> "$GITHUB_OUTPUT" if [ -n "${{ github.event.inputs.specs }}" ]; then echo "Scoped dispatch: ${{ github.event.inputs.specs }}" fi ;; esac - name: Record E2E execution context id: context env: E2E_TARGET_URL: ${{ vars.E2E_TARGET_URL }} run: | set -euo pipefail TARGET_URL="${E2E_TARGET_URL:-${PLAYWRIGHT_BASE_URL:-${BASE_URL:-http://localhost:3000}}}" jq -n \ --arg targetUrl "$TARGET_URL" \ --arg project "${{ steps.select.outputs.project }}" \ --arg specs "${{ steps.select.outputs.specs }}" \ --arg serverStart "consumer-managed-or-external" \ --arg serverStop "consumer-managed-or-external" \ --arg startedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --argjson timeoutMinutes 40 \ '{target_url: $targetUrl, project: $project, selected_specs: $specs, test_server_start: $serverStart, test_server_stop: $serverStop, timeout_minutes: $timeoutMinutes, started_at: $startedAt, outcome: "running"}' \ > e2e-regression-metadata.json echo "target_url=$TARGET_URL" >> "$GITHUB_OUTPUT" - name: Run E2E suite id: run env: PLAYWRIGHT_HTML_REPORTER_OPEN: never PLAYWRIGHT_JSON_OUTPUT_NAME: e2e-regression-results.json # Add your e2e_env values here as needed (DEVAUDIT_BASE_URL etc.) run: | set -uo pipefail PROJECT="${{ steps.select.outputs.project }}" SPECS="${{ steps.select.outputs.specs }}" if [ -n "$SPECS" ]; then timeout --signal=TERM --kill-after=60s 40m npx playwright test --project="$PROJECT" --reporter=json,html $SPECS else timeout --signal=TERM --kill-after=60s 40m npx playwright test --project="$PROJECT" --reporter=json,html fi STATUS=$? if [ "$STATUS" -eq 124 ]; then OUTCOME="timed_out" echo "::error::E2E regression exceeded its 40-minute execution budget. Partial evidence will be uploaded." elif [ "$STATUS" -eq 0 ]; then OUTCOME="passed" else OUTCOME="failed" fi jq --arg outcome "$OUTCOME" --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson exitCode "$STATUS" \ '. + {outcome: $outcome, completed_at: $completedAt, exit_code: $exitCode}' \ e2e-regression-metadata.json > e2e-regression-metadata.tmp mv e2e-regression-metadata.tmp e2e-regression-metadata.json exit "$STATUS" - uses: actions/upload-artifact@v7 if: always() with: name: e2e-regression-report path: | e2e-regression-results.json e2e-regression-metadata.json playwright-report/ test-results/ e2e-server.log server.log if-no-files-found: warn # ───────────────────────────────────────────────────────────── # Post-deployment auto-issue on regression failure. # # Catches regressions that slipped past the critical-tier PR gate. # Opens a high-priority issue tagging the merge commit + the # failing specs so the operator can triage within working hours. # No auto-revert — that's intentionally an operator decision. # ───────────────────────────────────────────────────────────── - name: Open hotfix issue on post-merge regression if: failure() && github.event_name == 'deployment_status' && github.event.deployment_status.state == 'success' env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail MERGE_SHA="${{ github.sha }}" MERGE_SHA_SHORT=$(echo "$MERGE_SHA" | cut -c1-7) RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" # Extract failing spec names from the JSON reporter output if available. FAILING="" if [ -f e2e-regression-results.json ]; then FAILING=$(jq -r ' [.. | objects | select(.status == "failed" or .status == "timedOut") | .title // empty] | unique | .[] ' e2e-regression-results.json 2>/dev/null | head -20 || true) fi if [ -z "$FAILING" ]; then FAILING="(see the failing run logs — could not parse spec titles from reporter output)" fi BODY=$(cat </error-context.md\` for page state at failure - [ ] Decide: hotfix on \`main\`, revert \`${MERGE_SHA_SHORT}\`, or accept-with-rationale if the failure is environmental - [ ] If the failing spec is a Must-tier candidate that should have caught this pre-merge, move it from \`e2e/\` to \`e2e/critical/\` so the next PR-to-main runs it **Auto-filed by:** \`e2e-regression.yml\` (devaudit#152 3-tier gating, v0.1.53+) EOF ) gh issue create \ --title "[hotfix] Post-merge regression on \`${MERGE_SHA_SHORT}\` — full E2E failed" \ --body "$BODY" \ --label "bug,priority:high"