#!/usr/bin/env bash # Module: pipeline-execution # Execution orchestration: stage retry logic, self-healing build-test loop, main pipeline run set -euo pipefail # Module guard [[ -n "${_MODULE_PIPELINE_EXECUTION_LOADED:-}" ]] && return 0; _MODULE_PIPELINE_EXECUTION_LOADED=1 # ─── Defaults (needed if sourced independently) ────────────────────────────── SCRIPT_DIR="${SCRIPT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" REPO_DIR="${REPO_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)}" PROJECT_ROOT="${PROJECT_ROOT:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}" STATE_DIR="${STATE_DIR:-$PROJECT_ROOT/.claude}" STATE_FILE="${STATE_FILE:-$STATE_DIR/pipeline-state.md}" ARTIFACTS_DIR="${ARTIFACTS_DIR:-$STATE_DIR/pipeline-artifacts}" # Variables referenced by execution functions (set by sw-pipeline.sh, defaults here for safety) BUILD_TEST_RETRIES="${BUILD_TEST_RETRIES:-2}" SELF_HEAL_COUNT="${SELF_HEAL_COUNT:-0}" STASHED_CHANGES="${STASHED_CHANGES:-false}" NOTIFICATION_ENABLED="${NOTIFICATION_ENABLED:-false}" HEARTBEAT_PID="${HEARTBEAT_PID:-}" # Ensure helpers are loaded [[ -f "$SCRIPT_DIR/lib/helpers.sh" ]] && source "$SCRIPT_DIR/lib/helpers.sh" 2>/dev/null || true [[ "$(type -t info 2>/dev/null)" == "function" ]] || info() { echo "$*"; } [[ "$(type -t warn 2>/dev/null)" == "function" ]] || warn() { echo "$*"; } [[ "$(type -t error 2>/dev/null)" == "function" ]] || error() { echo "$*" >&2; } [[ "$(type -t emit_event 2>/dev/null)" == "function" ]] || emit_event() { true; } # Ensure pipeline intelligence skip module is loaded (provides pipeline_should_skip_stage) # SCRIPT_DIR may point to scripts/ or scripts/lib/ depending on how this module was sourced if [[ -f "$SCRIPT_DIR/pipeline-intelligence-skip.sh" ]]; then source "$SCRIPT_DIR/pipeline-intelligence-skip.sh" 2>/dev/null || true elif [[ -f "$SCRIPT_DIR/lib/pipeline-intelligence-skip.sh" ]]; then source "$SCRIPT_DIR/lib/pipeline-intelligence-skip.sh" 2>/dev/null || true fi # ─── Stage Execution with Retry Logic ────────────────────────────── run_stage_with_retry() { local stage_id="$1" local max_retries max_retries=$(jq -r --arg id "$stage_id" '(.stages[] | select(.id == $id) | .config.retries) // 0' "$PIPELINE_CONFIG" 2>/dev/null) || true [[ -z "$max_retries" || "$max_retries" == "null" ]] && max_retries=0 local attempt=0 local prev_error_class="" while true; do if "stage_${stage_id}"; then return 0 fi # Capture error_class and error snippet for stage.failed / pipeline.completed events local error_class error_class=$(classify_error "$stage_id") LAST_STAGE_ERROR_CLASS="$error_class" LAST_STAGE_ERROR="" local _log_file="${ARTIFACTS_DIR}/${stage_id}-results.log" [[ ! -f "$_log_file" ]] && _log_file="${ARTIFACTS_DIR}/test-results.log" if [[ -f "$_log_file" ]]; then LAST_STAGE_ERROR=$(tail -20 "$_log_file" 2>/dev/null | grep -iE 'error|fail|exception|fatal' 2>/dev/null | head -1 | cut -c1-200 || true) fi attempt=$((attempt + 1)) # Critical fix: if plan stage already has a valid artifact, skip retry if [[ "$stage_id" == "plan" ]]; then local plan_artifact="${ARTIFACTS_DIR}/plan.md" if [[ -s "$plan_artifact" ]]; then local existing_lines existing_lines=$(wc -l < "$plan_artifact" 2>/dev/null | xargs) existing_lines="${existing_lines:-0}" if [[ "$existing_lines" -gt 10 ]]; then info "Plan already exists (${existing_lines} lines) — skipping retry, advancing" emit_event "retry.skipped_existing_artifact" \ "issue=${ISSUE_NUMBER:-0}" \ "stage=$stage_id" \ "artifact_lines=$existing_lines" return 0 fi fi fi if [[ "$attempt" -gt "$max_retries" ]]; then return 1 fi # Classify done above; decide whether retry makes sense emit_event "retry.classified" \ "issue=${ISSUE_NUMBER:-0}" \ "stage=$stage_id" \ "attempt=$attempt" \ "error_class=$error_class" case "$error_class" in infrastructure) info "Error classified as infrastructure (timeout/network/OOM) — retry makes sense" ;; configuration) error "Error classified as configuration (missing env/path) — skipping retry, escalating" emit_event "retry.escalated" \ "issue=${ISSUE_NUMBER:-0}" \ "stage=$stage_id" \ "reason=configuration_error" return 1 ;; logic) if [[ "$error_class" == "$prev_error_class" ]]; then error "Error classified as logic (assertion/type error) with same class — retry won't help without code change" emit_event "retry.skipped" \ "issue=${ISSUE_NUMBER:-0}" \ "stage=$stage_id" \ "reason=repeated_logic_error" return 1 fi warn "Error classified as logic — retrying once in case build fixes it" ;; *) info "Error classification: unknown — retrying" ;; esac prev_error_class="$error_class" if type db_save_reasoning_trace >/dev/null 2>&1; then local job_id="${SHIPWRIGHT_PIPELINE_ID:-$$}" local error_msg="${LAST_STAGE_ERROR:-$error_class}" db_save_reasoning_trace "$job_id" "retry_reasoning" \ "stage=$stage_id error=$error_msg" \ "Stage failed, analyzing error pattern before retry" \ "retry_strategy=self_heal" 0.6 2>/dev/null || true fi warn "Stage $stage_id failed (attempt $attempt/$((max_retries + 1)), class: $error_class) — retrying..." # Exponential backoff with jitter to avoid thundering herd local backoff=$((2 ** attempt)) [[ "$backoff" -gt 16 ]] && backoff=16 local jitter=$(( RANDOM % (backoff + 1) )) local total_sleep=$((backoff + jitter)) info "Backing off ${total_sleep}s before retry..." sleep "$total_sleep" # Write debugging context for the retry attempt to consume local _retry_ctx_file="${ARTIFACTS_DIR}/.retry-context-${stage_id}.md" { echo "## Previous Attempt Failed" echo "" echo "**Error classification:** ${error_class}" echo "**Attempt:** ${attempt} of $((max_retries + 1))" echo "" echo "### Error Output (last 30 lines)" echo '```' tail -30 "$_log_file" 2>/dev/null || echo "(no log available)" echo '```' echo "" # Check for existing artifacts that should be preserved local _existing_artifacts="" for _af in plan.md design.md test-results.log; do if [[ -s "${ARTIFACTS_DIR}/${_af}" ]]; then local _af_lines _af_lines=$(wc -l < "${ARTIFACTS_DIR}/${_af}" 2>/dev/null | xargs) _existing_artifacts="${_existing_artifacts} - ${_af} (${_af_lines} lines)\n" fi done if [[ -n "$_existing_artifacts" ]]; then echo "### Existing Artifacts (PRESERVE these)" echo -e "$_existing_artifacts" echo "These artifacts exist from previous successful stages. Use them as-is unless they are the source of the problem." echo "" fi # Adaptive: check if additional skills could help this retry if type skill_memory_get_recommendations >/dev/null 2>&1; then local _retry_skills _retry_skills=$(skill_memory_get_recommendations "${INTELLIGENCE_ISSUE_TYPE:-backend}" "$stage_id" 2>/dev/null || true) if [[ -n "$_retry_skills" ]]; then echo "### Skills Recommended by Learning System" echo "Based on historical success rates, these skills may improve the retry:" echo "- $(printf '%s' "$_retry_skills" | sed 's/,/\n- /g')" echo "" fi fi echo "### Investigation Required" echo "Before attempting a fix:" echo "1. Read the error output above carefully" echo "2. Identify the ROOT CAUSE — not just the symptom" echo "3. If previous artifacts exist and are correct, build on them" echo "4. If previous artifacts are flawed, explain what's wrong before fixing" } > "$_retry_ctx_file" 2>/dev/null || true emit_event "retry.context_written" \ "issue=${ISSUE_NUMBER:-0}" \ "stage=$stage_id" \ "attempt=$attempt" \ "context_file=$_retry_ctx_file" done } # ─── Self-Healing Build→Test Feedback Loop ───────────────────────── self_healing_build_test() { local cycle=0 local max_cycles="$BUILD_TEST_RETRIES" local last_test_error="" # Convergence tracking local prev_error_sig="" consecutive_same_error=0 local prev_fail_count=0 zero_convergence_streak=0 # Vitals-driven adaptive limit (preferred over static BUILD_TEST_RETRIES) if type pipeline_adaptive_limit >/dev/null 2>&1; then local _vitals_json="" if type pipeline_compute_vitals >/dev/null 2>&1; then _vitals_json=$(pipeline_compute_vitals "$STATE_FILE" "$ARTIFACTS_DIR" "${ISSUE_NUMBER:-}" 2>/dev/null) || true fi local vitals_limit vitals_limit=$(pipeline_adaptive_limit "build_test" "$_vitals_json" 2>/dev/null) || true if [[ -n "$vitals_limit" && "$vitals_limit" =~ ^[0-9]+$ && "$vitals_limit" -gt 0 ]]; then info "Vitals-driven build-test limit: ${max_cycles} → ${vitals_limit}" max_cycles="$vitals_limit" emit_event "vitals.adaptive_limit" \ "issue=${ISSUE_NUMBER:-0}" \ "context=build_test" \ "original=$BUILD_TEST_RETRIES" \ "vitals_limit=$vitals_limit" fi # Fallback: intelligence-based adaptive limits elif type composer_estimate_iterations >/dev/null 2>&1; then local estimated estimated=$(composer_estimate_iterations \ "${INTELLIGENCE_ANALYSIS:-{}}" \ "${HOME}/.shipwright/optimization/iteration-model.json" 2>/dev/null || echo "") if [[ -n "$estimated" && "$estimated" =~ ^[0-9]+$ && "$estimated" -gt 0 ]]; then max_cycles="$estimated" emit_event "intelligence.adaptive_iterations" \ "issue=${ISSUE_NUMBER:-0}" \ "estimated=$estimated" \ "original=$BUILD_TEST_RETRIES" fi fi # Fallback: adaptive cycle limits from optimization data if [[ "$max_cycles" == "$BUILD_TEST_RETRIES" ]]; then local _iter_model="${HOME}/.shipwright/optimization/iteration-model.json" if [[ -f "$_iter_model" ]]; then local adaptive_bt_limit adaptive_bt_limit=$(pipeline_adaptive_cycles "$max_cycles" "build_test" "0" "-1" 2>/dev/null) || true if [[ -n "$adaptive_bt_limit" && "$adaptive_bt_limit" =~ ^[0-9]+$ && "$adaptive_bt_limit" -gt 0 && "$adaptive_bt_limit" != "$max_cycles" ]]; then info "Adaptive build-test cycles: ${max_cycles} → ${adaptive_bt_limit}" max_cycles="$adaptive_bt_limit" fi fi fi while [[ "$cycle" -le "$max_cycles" ]]; do cycle=$((cycle + 1)) if [[ "$cycle" -gt 1 ]]; then SELF_HEAL_COUNT=$((SELF_HEAL_COUNT + 1)) echo "" echo -e "${YELLOW}${BOLD}━━━ Self-Healing Cycle ${cycle}/$((max_cycles + 1)) ━━━${RESET}" info "Feeding test failure back to build loop..." if [[ -n "$ISSUE_NUMBER" ]]; then gh_comment_issue "$ISSUE_NUMBER" "🔄 **Self-healing cycle ${cycle}** — rebuilding with error context" 2>/dev/null || true fi # Reset build/test stage statuses for retry set_stage_status "build" "retrying" set_stage_status "test" "pending" fi # ── Run Build Stage ── echo "" echo -e "${CYAN}${BOLD}▸ Stage: build${RESET} ${DIM}[cycle ${cycle}]${RESET}" CURRENT_STAGE_ID="build" # Inject error context on retry cycles if [[ "$cycle" -gt 1 && -n "$last_test_error" ]]; then # Query memory for known fixes local _memory_fix="" if type memory_closed_loop_inject >/dev/null 2>&1; then local _error_sig_short _error_sig_short=$(echo "$last_test_error" | head -3 || echo "") _memory_fix=$(memory_closed_loop_inject "$_error_sig_short" 2>/dev/null) || true fi local memory_prefix="" if [[ -n "$_memory_fix" ]]; then info "Memory suggests fix: $(echo "$_memory_fix" | head -1)" memory_prefix="KNOWN FIX (from past success): ${_memory_fix} " fi # Temporarily augment the goal with error context local original_goal="$GOAL" GOAL="$GOAL ${memory_prefix}IMPORTANT — Previous build attempt failed tests. Fix these errors: $last_test_error Focus on fixing the failing tests while keeping all passing tests working." update_status "running" "build" record_stage_start "build" type audit_emit >/dev/null 2>&1 && audit_emit "stage.start" "stage=build" || true local build_start_epoch build_start_epoch=$(date +%s) if run_stage_with_retry "build"; then mark_stage_complete "build" local timing timing=$(get_stage_timing "build") local build_dur_s=$(( $(date +%s) - build_start_epoch )) type audit_emit >/dev/null 2>&1 && audit_emit "stage.complete" "stage=build" "verdict=pass" "duration_s=${build_dur_s}" || true success "Stage ${BOLD}build${RESET} complete ${DIM}(${timing})${RESET}" if type pipeline_emit_progress_snapshot >/dev/null 2>&1 && [[ -n "${ISSUE_NUMBER:-}" ]]; then local _diff_count _diff_count=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1) || true local _snap_files _snap_error _snap_files=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1 || true) _snap_files="${_snap_files:-0}" _snap_error=$(tail -1 "$ARTIFACTS_DIR/error-log.jsonl" 2>/dev/null | jq -r '.error // ""' 2>/dev/null || true) _snap_error="${_snap_error:-}" pipeline_emit_progress_snapshot "${ISSUE_NUMBER}" "${CURRENT_STAGE_ID:-build}" "${cycle:-0}" "${_diff_count:-0}" "${_snap_files}" "${_snap_error}" 2>/dev/null || true fi else mark_stage_failed "build" local build_dur_s=$(( $(date +%s) - build_start_epoch )) type audit_emit >/dev/null 2>&1 && audit_emit "stage.complete" "stage=build" "verdict=fail" "duration_s=${build_dur_s}" || true GOAL="$original_goal" return 1 fi GOAL="$original_goal" else update_status "running" "build" record_stage_start "build" type audit_emit >/dev/null 2>&1 && audit_emit "stage.start" "stage=build" || true local build_start_epoch build_start_epoch=$(date +%s) if run_stage_with_retry "build"; then mark_stage_complete "build" local timing timing=$(get_stage_timing "build") local build_dur_s=$(( $(date +%s) - build_start_epoch )) type audit_emit >/dev/null 2>&1 && audit_emit "stage.complete" "stage=build" "verdict=pass" "duration_s=${build_dur_s}" || true success "Stage ${BOLD}build${RESET} complete ${DIM}(${timing})${RESET}" if type pipeline_emit_progress_snapshot >/dev/null 2>&1 && [[ -n "${ISSUE_NUMBER:-}" ]]; then local _diff_count _diff_count=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1) || true local _snap_files _snap_error _snap_files=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1 || true) _snap_files="${_snap_files:-0}" _snap_error=$(tail -1 "$ARTIFACTS_DIR/error-log.jsonl" 2>/dev/null | jq -r '.error // ""' 2>/dev/null || true) _snap_error="${_snap_error:-}" pipeline_emit_progress_snapshot "${ISSUE_NUMBER}" "${CURRENT_STAGE_ID:-build}" "${cycle:-0}" "${_diff_count:-0}" "${_snap_files}" "${_snap_error}" 2>/dev/null || true fi else mark_stage_failed "build" local build_dur_s=$(( $(date +%s) - build_start_epoch )) type audit_emit >/dev/null 2>&1 && audit_emit "stage.complete" "stage=build" "verdict=fail" "duration_s=${build_dur_s}" || true return 1 fi fi # ── Run Test Stage ── echo "" echo -e "${CYAN}${BOLD}▸ Stage: test${RESET} ${DIM}[cycle ${cycle}]${RESET}" CURRENT_STAGE_ID="test" update_status "running" "test" record_stage_start "test" if run_stage_with_retry "test"; then mark_stage_complete "test" local timing timing=$(get_stage_timing "test") success "Stage ${BOLD}test${RESET} complete ${DIM}(${timing})${RESET}" emit_event "convergence.tests_passed" \ "issue=${ISSUE_NUMBER:-0}" \ "cycle=$cycle" if type pipeline_emit_progress_snapshot >/dev/null 2>&1 && [[ -n "${ISSUE_NUMBER:-}" ]]; then local _diff_count _diff_count=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1) || true local _snap_files _snap_error _snap_files=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1 || true) _snap_files="${_snap_files:-0}" _snap_error=$(tail -1 "$ARTIFACTS_DIR/error-log.jsonl" 2>/dev/null | jq -r '.error // ""' 2>/dev/null || true) _snap_error="${_snap_error:-}" pipeline_emit_progress_snapshot "${ISSUE_NUMBER}" "${CURRENT_STAGE_ID:-test}" "${cycle:-0}" "${_diff_count:-0}" "${_snap_files}" "${_snap_error}" 2>/dev/null || true fi # Record fix outcome when tests pass after a retry with memory injection (pipeline path) if [[ "$cycle" -gt 1 && -n "${last_test_error:-}" ]] && [[ -x "$SCRIPT_DIR/sw-memory.sh" ]]; then local _sig _sig=$(echo "$last_test_error" | head -3 | tr '\n' ' ' | sed 's/^ *//;s/ *$//') [[ -n "$_sig" ]] && bash "$SCRIPT_DIR/sw-memory.sh" fix-outcome "$_sig" "true" "true" 2>/dev/null || true fi return 0 # Tests passed! fi # Tests failed — capture error for next cycle local test_log="$ARTIFACTS_DIR/test-results.log" last_test_error=$(tail -30 "$test_log" 2>/dev/null || echo "Test command failed with no output") mark_stage_failed "test" # ── Convergence Detection ── # Hash the error output to detect repeated failures local error_sig error_sig=$(echo "$last_test_error" | shasum -a 256 2>/dev/null | cut -c1-16 || echo "unknown") # Count failing tests (extract from common patterns) local current_fail_count=0 current_fail_count=$(grep -ciE 'fail|error|FAIL' "$test_log" 2>/dev/null || true) current_fail_count="${current_fail_count:-0}" if [[ "$error_sig" == "$prev_error_sig" ]]; then consecutive_same_error=$((consecutive_same_error + 1)) else consecutive_same_error=1 fi prev_error_sig="$error_sig" # Check: same error 3 times consecutively → stuck if [[ "$consecutive_same_error" -ge 3 ]]; then error "Convergence: stuck on same error for 3 consecutive cycles — exiting early" emit_event "convergence.stuck" \ "issue=${ISSUE_NUMBER:-0}" \ "cycle=$cycle" \ "error_sig=$error_sig" \ "consecutive=$consecutive_same_error" notify "Build Convergence" "Stuck on unfixable error after ${cycle} cycles" "error" return 1 fi # Track convergence rate: did we reduce failures? if [[ "$cycle" -gt 1 && "$prev_fail_count" -gt 0 ]]; then if [[ "$current_fail_count" -ge "$prev_fail_count" ]]; then zero_convergence_streak=$((zero_convergence_streak + 1)) else zero_convergence_streak=0 fi # Check: zero convergence for 2 consecutive iterations → plateau if [[ "$zero_convergence_streak" -ge 2 ]]; then error "Convergence: no progress for 2 consecutive cycles (${current_fail_count} failures remain) — exiting early" emit_event "convergence.plateau" \ "issue=${ISSUE_NUMBER:-0}" \ "cycle=$cycle" \ "fail_count=$current_fail_count" \ "streak=$zero_convergence_streak" notify "Build Convergence" "No progress after ${cycle} cycles — plateau reached" "error" return 1 fi fi prev_fail_count="$current_fail_count" info "Convergence: error_sig=${error_sig:0:8} repeat=${consecutive_same_error} failures=${current_fail_count} no_progress=${zero_convergence_streak}" if [[ "$cycle" -le "$max_cycles" ]]; then warn "Tests failed — will attempt self-healing (cycle $((cycle + 1))/$((max_cycles + 1)))" notify "Self-Healing" "Tests failed on cycle ${cycle}, retrying..." "warn" fi done error "Self-healing exhausted after $((max_cycles + 1)) cycles" notify "Self-Healing Failed" "Tests still failing after $((max_cycles + 1)) build-test cycles" "error" return 1 } # ─── Auto-Rebase Before PR ───────────────────────────────────────── auto_rebase() { info "Syncing with ${BASE_BRANCH}..." # Fetch latest git fetch origin "$BASE_BRANCH" --quiet 2>/dev/null || { warn "Could not fetch origin/${BASE_BRANCH}" return 0 } # Check if rebase is needed local behind behind=$(git rev-list --count "HEAD..origin/${BASE_BRANCH}" 2>/dev/null || echo "0") if [[ "$behind" -eq 0 ]]; then success "Already up to date with ${BASE_BRANCH}" return 0 fi info "Rebasing onto origin/${BASE_BRANCH} ($behind commits behind)..." if git rebase "origin/${BASE_BRANCH}" --quiet 2>/dev/null; then success "Rebase successful" else warn "Rebase conflict detected — aborting rebase" git rebase --abort 2>/dev/null || true warn "Falling back to merge..." if git merge "origin/${BASE_BRANCH}" --no-edit --quiet 2>/dev/null; then success "Merge successful" else git merge --abort 2>/dev/null || true error "Both rebase and merge failed — manual intervention needed" return 1 fi fi } # ─── Main Pipeline Orchestration ─────────────────────────────────── run_pipeline() { # Rotate event log if needed (standalone mode) rotate_event_log_if_needed # Initialize audit trail for this pipeline run if type audit_init >/dev/null 2>&1; then audit_init || true fi local stages stages=$(jq -c '.stages[]' "$PIPELINE_CONFIG" 2>/dev/null) local stage_count enabled_count stage_count=$(jq '.stages | length' "$PIPELINE_CONFIG" 2>/dev/null) enabled_count=$(jq '[.stages[] | select(.enabled == true)] | length' "$PIPELINE_CONFIG" 2>/dev/null) local completed=0 # Check which stages are enabled to determine if we use the self-healing loop local build_enabled test_enabled build_enabled=$(jq -r '.stages[] | select(.id == "build") | .enabled' "$PIPELINE_CONFIG" 2>/dev/null) test_enabled=$(jq -r '.stages[] | select(.id == "test") | .enabled' "$PIPELINE_CONFIG" 2>/dev/null) local use_self_healing=false if [[ "$build_enabled" == "true" && "$test_enabled" == "true" && "$BUILD_TEST_RETRIES" -gt 0 ]]; then use_self_healing=true fi while IFS= read -r -u 3 stage; do local id enabled gate id=$(echo "$stage" | jq -r '.id' 2>/dev/null) enabled=$(echo "$stage" | jq -r '.enabled' 2>/dev/null) gate=$(echo "$stage" | jq -r '.gate' 2>/dev/null) CURRENT_STAGE_ID="$id" # Human intervention: check for skip-stage directive if [[ -f "$ARTIFACTS_DIR/skip-stage.txt" ]]; then local skip_list skip_list="$(cat "$ARTIFACTS_DIR/skip-stage.txt" 2>/dev/null || true)" if echo "$skip_list" | grep -qx "$id" 2>/dev/null; then info "Stage ${BOLD}${id}${RESET} skipped by human directive" emit_event "stage.skipped" "issue=${ISSUE_NUMBER:-0}" "stage=$id" "reason=human_skip" # Remove this stage from the skip file local tmp_skip tmp_skip="$(mktemp)" || { warn "mktemp failed"; continue; } # shellcheck disable=SC2064 # intentional expansion at definition time trap "rm -f '$tmp_skip'" RETURN grep -vx "$id" "$ARTIFACTS_DIR/skip-stage.txt" > "$tmp_skip" 2>/dev/null || true mv "$tmp_skip" "$ARTIFACTS_DIR/skip-stage.txt" continue fi fi # Human intervention: check for human message if [[ -f "$ARTIFACTS_DIR/human-message.txt" ]]; then local human_msg human_msg="$(cat "$ARTIFACTS_DIR/human-message.txt" 2>/dev/null || true)" if [[ -n "$human_msg" ]]; then echo "" echo -e " ${PURPLE}${BOLD}💬 Human message:${RESET} $human_msg" emit_event "pipeline.human_message" "issue=${ISSUE_NUMBER:-0}" "stage=$id" "message=$human_msg" rm -f "$ARTIFACTS_DIR/human-message.txt" fi fi if [[ "$enabled" != "true" ]]; then echo -e " ${DIM}○ ${id} — skipped (disabled)${RESET}" continue fi # Intelligence: evaluate whether to skip this stage (after intake, which populates ISSUE_LABELS) if [[ "$id" != "intake" ]] && type pipeline_should_skip_stage >/dev/null 2>&1; then local skip_reason="" skip_reason=$(pipeline_should_skip_stage "$id" 2>/dev/null) || true if [[ -n "$skip_reason" ]]; then echo -e " ${DIM}○ ${id} — skipped (intelligence: ${skip_reason})${RESET}" set_stage_status "$id" "complete" completed=$((completed + 1)) continue fi fi local stage_status stage_status=$(get_stage_status "$id") if [[ "$stage_status" == "complete" ]]; then echo -e " ${GREEN}✓ ${id}${RESET} ${DIM}— already complete${RESET}" completed=$((completed + 1)) continue fi # CI resume: skip stages marked as completed from previous run if [[ -n "${COMPLETED_STAGES:-}" ]] && echo "$COMPLETED_STAGES" | tr ',' '\n' | grep -qx "$id"; then # Verify artifacts survived the merge — regenerate if missing if verify_stage_artifacts "$id"; then echo -e " ${GREEN}✓ ${id}${RESET} ${DIM}— skipped (CI resume)${RESET}" set_stage_status "$id" "complete" completed=$((completed + 1)) emit_event "stage.skipped" "issue=${ISSUE_NUMBER:-0}" "stage=$id" "reason=ci_resume" continue else warn "Stage $id marked complete but artifacts missing — regenerating" emit_event "stage.artifact_miss" "issue=${ISSUE_NUMBER:-0}" "stage=$id" fi fi # Self-healing build→test loop: when we hit build, run both together if [[ "$id" == "build" && "$use_self_healing" == "true" ]]; then # TDD: generate tests before build when enabled if [[ "${TDD_ENABLED:-false}" == "true" || "${PIPELINE_TDD:-}" == "true" ]]; then stage_test_first || true fi # Gate check for build local build_gate build_gate=$(echo "$stage" | jq -r '.gate' 2>/dev/null) if [[ "$build_gate" == "approve" && "$SKIP_GATES" != "true" ]]; then show_stage_preview "build" local answer="" if [[ -t 0 ]]; then read -rp " Proceed with build+test (self-healing)? [Y/n] " answer || true fi if [[ "$answer" =~ ^[Nn] ]]; then update_status "paused" "build" info "Pipeline paused. Resume with: ${DIM}shipwright pipeline resume${RESET}" return 0 fi fi if self_healing_build_test; then completed=$((completed + 2)) # Both build and test # Intelligence: reassess complexity after build+test local reassessment reassessment=$(pipeline_reassess_complexity 2>/dev/null) || true if [[ -n "$reassessment" && "$reassessment" != "as_expected" ]]; then info "Complexity reassessment: ${reassessment}" fi else update_status "failed" "test" error "Pipeline failed: build→test self-healing exhausted" return 1 fi continue fi # TDD: generate tests before build when enabled (non-self-healing path) if [[ "$id" == "build" && "$use_self_healing" != "true" ]] && [[ "${TDD_ENABLED:-false}" == "true" || "${PIPELINE_TDD:-}" == "true" ]]; then stage_test_first || true fi # Skip test if already handled by self-healing loop if [[ "$id" == "test" && "$use_self_healing" == "true" ]]; then stage_status=$(get_stage_status "test") if [[ "$stage_status" == "complete" ]]; then echo -e " ${GREEN}✓ test${RESET} ${DIM}— completed in build→test loop${RESET}" fi continue fi # Gate check if [[ "$gate" == "approve" && "$SKIP_GATES" != "true" ]]; then show_stage_preview "$id" local answer="" if [[ -t 0 ]]; then read -rp " Proceed with ${id}? [Y/n] " answer || true else # Non-interactive: auto-approve (shouldn't reach here if headless detection works) info "Non-interactive mode — auto-approving ${id}" fi if [[ "$answer" =~ ^[Nn] ]]; then update_status "paused" "$id" info "Pipeline paused at ${BOLD}$id${RESET}. Resume with: ${DIM}shipwright pipeline resume${RESET}" return 0 fi fi # Budget enforcement check (skip with --ignore-budget) if [[ "$IGNORE_BUDGET" != "true" ]] && [[ -x "$SCRIPT_DIR/sw-cost.sh" ]]; then local budget_rc=0 bash "$SCRIPT_DIR/sw-cost.sh" check-budget 2>/dev/null || budget_rc=$? if [[ "$budget_rc" -eq 2 ]]; then warn "Daily budget exceeded — pausing pipeline before stage ${BOLD}$id${RESET}" warn "Resume with --ignore-budget to override, or wait until tomorrow" emit_event "pipeline.budget_paused" "issue=${ISSUE_NUMBER:-0}" "stage=$id" update_status "paused" "$id" return 0 fi fi # Intelligence: per-stage model routing (UCB1 when DB has data, else A/B testing) local recommended_model="" from_ucb1=false if type ucb1_select_model >/dev/null 2>&1; then recommended_model=$(ucb1_select_model "$id" 2>/dev/null || echo "") [[ -n "$recommended_model" ]] && from_ucb1=true fi if [[ -z "$recommended_model" ]] && type intelligence_recommend_model >/dev/null 2>&1; then local stage_complexity="${INTELLIGENCE_COMPLEXITY:-5}" local budget_remaining="" if [[ -x "$SCRIPT_DIR/sw-cost.sh" ]]; then budget_remaining=$(bash "$SCRIPT_DIR/sw-cost.sh" remaining-budget 2>/dev/null || echo "") fi local recommended_json recommended_json=$(intelligence_recommend_model "$id" "$stage_complexity" "$budget_remaining" 2>/dev/null || echo "") recommended_model=$(echo "$recommended_json" | jq -r '.model // empty' 2>/dev/null || echo "") fi if [[ -n "$recommended_model" && "$recommended_model" != "null" ]]; then if [[ "$from_ucb1" == "true" ]]; then # UCB1 already balances exploration/exploitation — use directly export CLAUDE_MODEL="$recommended_model" emit_event "intelligence.model_ucb1" \ "issue=${ISSUE_NUMBER:-0}" \ "stage=$id" \ "model=$recommended_model" else # A/B testing for intelligence recommendation local ab_ratio=20 local daemon_cfg="${PROJECT_ROOT}/.claude/daemon-config.json" if [[ -f "$daemon_cfg" ]]; then local cfg_ratio cfg_ratio=$(jq -r '.intelligence.ab_test_ratio // 0.2' "$daemon_cfg" 2>/dev/null || echo "0.2") ab_ratio=$(awk -v r="$cfg_ratio" 'BEGIN{printf "%d", r * 100}' 2>/dev/null || echo "20") fi local routing_file="${HOME}/.shipwright/optimization/model-routing.json" local use_recommended=false local ab_group="control" if [[ -f "$routing_file" ]]; then local stage_samples total_samples stage_samples=$(jq -r --arg s "$id" '.routes[$s].sonnet_samples // .[$s].sonnet_samples // 0' "$routing_file" 2>/dev/null || echo "0") total_samples=$(jq -r --arg s "$id" '((.routes[$s].sonnet_samples // .[$s].sonnet_samples // 0) + (.routes[$s].opus_samples // .[$s].opus_samples // 0))' "$routing_file" 2>/dev/null || echo "0") if [[ "${total_samples:-0}" -ge 50 ]]; then use_recommended=true ab_group="graduated" fi fi if [[ "$use_recommended" != "true" ]]; then local roll=$((RANDOM % 100)) if [[ "$roll" -lt "$ab_ratio" ]]; then use_recommended=true ab_group="experiment" fi fi if [[ "$use_recommended" == "true" ]]; then export CLAUDE_MODEL="$recommended_model" else export CLAUDE_MODEL="$(_smart_model default sonnet)" fi emit_event "intelligence.model_ab" \ "issue=${ISSUE_NUMBER:-0}" \ "stage=$id" \ "recommended=$recommended_model" \ "applied=$CLAUDE_MODEL" \ "ab_group=$ab_group" \ "ab_ratio=$ab_ratio" fi fi echo "" echo -e "${CYAN}${BOLD}▸ Stage: ${id}${RESET} ${DIM}[$((completed + 1))/${enabled_count}]${RESET}" update_status "running" "$id" record_stage_start "$id" local stage_start_epoch stage_start_epoch=$(now_epoch) emit_event "stage.started" "issue=${ISSUE_NUMBER:-0}" "stage=$id" # Mark GitHub Check Run as in-progress if [[ "${NO_GITHUB:-false}" != "true" ]] && type gh_checks_stage_update >/dev/null 2>&1; then gh_checks_stage_update "$id" "in_progress" "" "Stage $id started" 2>/dev/null || true fi # Audit: stage start if type audit_emit >/dev/null 2>&1; then audit_emit "stage.start" "stage=$id" || true fi local stage_model_used="${CLAUDE_MODEL:-${MODEL:-opus}}" if run_stage_with_retry "$id"; then mark_stage_complete "$id" completed=$((completed + 1)) # Capture project pattern after intake (for memory context in later stages) if [[ "$id" == "intake" ]] && [[ -x "$SCRIPT_DIR/sw-memory.sh" ]]; then (cd "$REPO_DIR" && bash "$SCRIPT_DIR/sw-memory.sh" pattern "project" "{}" 2>/dev/null) || true fi local timing stage_dur_s timing=$(get_stage_timing "$id") stage_dur_s=$(( $(now_epoch) - stage_start_epoch )) success "Stage ${BOLD}$id${RESET} complete ${DIM}(${timing})${RESET}" emit_event "stage.completed" "issue=${ISSUE_NUMBER:-0}" "stage=$id" "duration_s=$stage_dur_s" "result=success" # Audit: stage complete if type audit_emit >/dev/null 2>&1; then audit_emit "stage.complete" "stage=$id" "verdict=pass" \ "duration_s=${stage_dur_s:-0}" || true fi # Emit vitals snapshot on every stage transition (not just build/test) if type pipeline_emit_progress_snapshot >/dev/null 2>&1 && [[ -n "${ISSUE_NUMBER:-}" ]]; then pipeline_emit_progress_snapshot "${ISSUE_NUMBER}" "$id" "0" "0" "0" "" 2>/dev/null || true fi # Record model outcome for UCB1 learning type record_model_outcome >/dev/null 2>&1 && record_model_outcome "$stage_model_used" "$id" 1 "$stage_dur_s" 0 2>/dev/null || true # Broadcast discovery for cross-pipeline learning if [[ -x "$SCRIPT_DIR/sw-discovery.sh" ]]; then local _disc_cat _disc_patterns _disc_text _disc_cat="$id" case "$id" in plan) _disc_patterns="*.md"; _disc_text="Plan completed: ${GOAL:-goal}" ;; design) _disc_patterns="*.md,*.ts,*.tsx,*.js"; _disc_text="Design completed for ${GOAL:-goal}" ;; build) _disc_patterns="src/*,*.ts,*.tsx,*.js"; _disc_text="Build completed" ;; test) _disc_patterns="*.test.*,*_test.*"; _disc_text="Tests passed" ;; review) _disc_patterns="*.md,*.ts,*.tsx"; _disc_text="Review completed" ;; *) _disc_patterns="*"; _disc_text="Stage $id completed" ;; esac bash "$SCRIPT_DIR/sw-discovery.sh" broadcast "$_disc_cat" "$_disc_patterns" "$_disc_text" "" 2>/dev/null || true fi # Log model used for prediction feedback echo "${id}|${stage_model_used}|true" >> "${ARTIFACTS_DIR}/model-routing.log" else mark_stage_failed "$id" local stage_dur_s stage_dur_s=$(( $(now_epoch) - stage_start_epoch )) error "Pipeline failed at stage: ${BOLD}$id${RESET}" update_status "failed" "$id" emit_event "stage.failed" \ "issue=${ISSUE_NUMBER:-0}" \ "stage=$id" \ "duration_s=$stage_dur_s" \ "error=${LAST_STAGE_ERROR:-unknown}" \ "error_class=${LAST_STAGE_ERROR_CLASS:-unknown}" # Audit: stage failed if type audit_emit >/dev/null 2>&1; then audit_emit "stage.complete" "stage=$id" "verdict=fail" \ "duration_s=${stage_dur_s:-0}" || true fi # Emit vitals snapshot on failure too if type pipeline_emit_progress_snapshot >/dev/null 2>&1 && [[ -n "${ISSUE_NUMBER:-}" ]]; then pipeline_emit_progress_snapshot "${ISSUE_NUMBER}" "$id" "0" "0" "0" "${LAST_STAGE_ERROR:-unknown}" 2>/dev/null || true fi # Log model used for prediction feedback echo "${id}|${stage_model_used}|false" >> "${ARTIFACTS_DIR}/model-routing.log" # Record model outcome for UCB1 learning type record_model_outcome >/dev/null 2>&1 && record_model_outcome "$stage_model_used" "$id" 0 "$stage_dur_s" 0 2>/dev/null || true # Cancel any remaining in_progress check runs pipeline_cancel_check_runs 2>/dev/null || true return 1 fi done 3<<< "$stages" # Pipeline complete! update_status "complete" "" PIPELINE_STAGES_PASSED="$completed" PIPELINE_SLOWEST_STAGE="" if type get_slowest_stage >/dev/null 2>&1; then PIPELINE_SLOWEST_STAGE=$(get_slowest_stage 2>/dev/null || true) fi local total_dur="" if [[ -n "$PIPELINE_START_EPOCH" ]]; then total_dur=$(format_duration $(( $(now_epoch) - PIPELINE_START_EPOCH ))) fi echo "" echo -e "${GREEN}${BOLD}═══════════════════════════════════════════════════════════════════${RESET}" success "Pipeline complete! ${completed}/${enabled_count} stages passed in ${total_dur:-unknown}" echo -e "${GREEN}${BOLD}═══════════════════════════════════════════════════════════════════${RESET}" # Show summary echo "" if [[ -f "$ARTIFACTS_DIR/pr-url.txt" ]]; then echo -e " ${BOLD}PR:${RESET} $(cat "$ARTIFACTS_DIR/pr-url.txt")" fi echo -e " ${BOLD}Branch:${RESET} $GIT_BRANCH" [[ -n "${GITHUB_ISSUE:-}" ]] && echo -e " ${BOLD}Issue:${RESET} $GITHUB_ISSUE" echo -e " ${BOLD}Duration:${RESET} $total_dur" echo -e " ${BOLD}Artifacts:${RESET} $ARTIFACTS_DIR/" echo "" # Capture learnings to memory (success or failure) if [[ -x "$SCRIPT_DIR/sw-memory.sh" ]]; then bash "$SCRIPT_DIR/sw-memory.sh" capture "$STATE_FILE" "$ARTIFACTS_DIR" 2>/dev/null || true fi # Final GitHub progress update if [[ -n "$ISSUE_NUMBER" ]]; then local body body=$(gh_build_progress_body) gh_update_progress "$body" fi # Post-completion cleanup pipeline_post_completion_cleanup }