#!/usr/bin/env bash
# cluster-up — sema-up 的 full-cluster 装配(Phase 1b as-built 的编码化)。
# 由 sema-up.sh 在 PROFILE=full-cluster 时调用,也可单独跑(env 见下)。
# 流程:机器互达检 → k3s HA(embedded etcd)→ VXLAN 守卫 → Longhorn → kata-deploy →
#       CNPG operator → 镜像分发 → 渲染 sema-stack-cluster.yaml → apply → token 复制 →
#       建桶 → smoke。全程幂等 check-then-apply;凭据不轮换;密码不入日志。
#
# 输入(sema-up.sh 解析 machines.yaml 后注入):
#   MACHINE_COUNT + MACHINE_<i>_{HOST,USER,SSH_KEY,NAME,ROLE,PRIVATE_IP,PUBLIC_IFACE}
#   WORK(状态目录,.env 在此)REGION PROJECT
#   SEMA_SERVER_IMAGE(registry 引用)或 SEMA_SERVER_IMAGE_TAR(本地 docker save tar[.gz])二选一
#   MODEL_GATEWAY_BASEURL MODEL_ID MODEL_API_KEY
#   ALLOW_SMALL_CLUSTER=true 放行 <3 机(dev/CI 单节点验流程;生产必须 >=3 奇数)
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
log() { printf '[cluster-up] %s\n' "$*" >&2; }
die() { log "⛔ $*"; exit 1; }

: "${WORK:?WORK 未设置(状态目录)}"
: "${MACHINE_COUNT:?MACHINE_COUNT 未设置(machines.yaml 未解析?)}"
REGION="${REGION:-global}"
K3S_CHANNEL="${K3S_CHANNEL:-stable}"
CNPG_VERSION="${CNPG_VERSION:-1.30.0}"   # ⚠ in-tree barman 1.31 移除;升版前先迁 barman-cloud plugin
LONGHORN_VERSION="${LONGHORN_VERSION:-1.10.0}"
KATA_VERSION="${KATA_VERSION:-3.21.0}"
NODEPORT=30890

# <3 机=建议不强求(clay 2026-07-12:有建议的 3 节点配置,仍能处理;某台机器过小才拦——
# 那个由 0/8 每机预检的 8G/40G 硬底线负责)。ALLOW_SMALL_CLUSTER 保留兼容,不再必需。
if [ "$MACHINE_COUNT" -lt 3 ]; then
  log "⚠ 建议:准生产=3 机(embedded etcd HA+PG 反亲和);当前 $MACHINE_COUNT 机能跑但 etcd 无 HA(单点/双机 quorum 脆),形态按机器数缩放"
fi
if [ $((MACHINE_COUNT % 2)) -eq 0 ]; then
  log "⚠ 偶数台($MACHINE_COUNT)etcd 容错不增反险(quorum=$((MACHINE_COUNT/2+1))),建议奇数"
fi

# ── ssh 帮手(role 无关,i 从 0)────────────────────────────────────────────────
mvar() { local n="MACHINE_$1_$2"; printf '%s' "${!n:-}"; }
mssh() { # mssh <i> <cmd…>;远端跑 sudo bash -s 读 stdin 时用 msshs
  local i="$1"; shift
  local key; key="$(mvar "$i" SSH_KEY)"
  ssh -o ConnectTimeout=15 -o ServerAliveInterval=15 -o ServerAliveCountMax=8 \
    -o StrictHostKeyChecking=accept-new ${key:+-i "${key/#\~/$HOME}"} \
    "$(mvar "$i" USER)@$(mvar "$i" HOST)" "$@"
}
mscp() { # mscp <i> <local> <remote>
  local i="$1"; local key; key="$(mvar "$i" SSH_KEY)"
  scp -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new ${key:+-i "${key/#\~/$HOME}"} \
    "$2" "$(mvar "$i" USER)@$(mvar "$i" HOST):$3"
}
# 远端 root 执行:user=root 直跑,否则要求 passwordless sudo(预检里验)
rsudo() { local i="$1"; shift; if [ "$(mvar "$i" USER)" = "root" ]; then mssh "$i" bash -s -- "$@"; else mssh "$i" sudo bash -s -- "$@"; fi; }
# 远端单命令(stdin 归命令用,如 kubectl apply -f -)
rrun() { local i="$1"; shift; if [ "$(mvar "$i" USER)" = "root" ]; then mssh "$i" "$@"; else mssh "$i" sudo "$@"; fi; }

# ── 0. 互达 + sudo + 硬件预检(在目标机上,§3d-2:错误带原因+建议)+ hostname 告警 ──
log "0/8 机器预检($MACHINE_COUNT 台)"
declare -a SYS_HOSTNAMES=()
for ((i=0; i<MACHINE_COUNT; i++)); do
  out=$(rsudo "$i" <<'EOF'
set -e
command -v curl >/dev/null || { echo "FAIL curl 缺失——建议:apt-get install -y curl"; exit 1; }
[ "$(uname -m)" = "x86_64" ] || [ "$(uname -m)" = "aarch64" ] || { echo "FAIL 架构 $(uname -m) 不支持——建议:x86_64/aarch64 机器"; exit 1; }
ram=$(awk '/MemTotal/{printf "%d", $2/1024}' /proc/meminfo)
[ "$ram" -ge 8192 ] || { echo "FAIL 内存 ${ram}MB < 8G——建议:full-cluster 节点至少 8G(数据面+沙箱同机)"; exit 1; }
disk=$(df -m / | awk 'NR==2{print $4}')
[ "$disk" -ge 40960 ] || { echo "FAIL 根盘可用 ${disk}MB < 40G——建议:Longhorn/镜像要空间,清盘或换机"; exit 1; }
[ -e /dev/kvm ] || echo "WARN 无 /dev/kvm——kata VM 沙箱在本节点不可用(建议:开嵌套虚拟化/裸机;或全集群改 e2b lane)"
hostname
EOF
  ) || die "机器 $i($(mvar "$i" HOST))预检失败:${out:-ssh/sudo 不通——需要 key 互信 + passwordless sudo}"
  case "$out" in *WARN*) log "  ⚠ $(mvar "$i" HOST):${out%%$'\n'*}" ;; esac
  SYS_HOSTNAMES+=("$(printf '%s\n' "$out" | tail -1)")
done
if [ "$(printf '%s\n' "${SYS_HOSTNAMES[@]}" | sort -u | wc -l)" -lt "$MACHINE_COUNT" ]; then
  log "⚠ 系统 hostname 有重复(${SYS_HOSTNAMES[*]})——裸机批次常见;k3s 一律用唯一 --node-name 注册,不依赖系统名(Phase 1b etcd 撞名教训)"
fi

# region=auto:从目标机(不是操作员机)探测网络亲和
if [ "$REGION" = "auto" ]; then
  REGION=$(rsudo 0 <<'EOF'
a=$(curl -m3 -so /dev/null -w '%{time_total}' https://mirrors.aliyun.com 2>/dev/null || echo 99)
b=$(curl -m3 -so /dev/null -w '%{time_total}' https://registry.npmjs.org 2>/dev/null || echo 99)
awk -v a="$a" -v b="$b" 'BEGIN{print (a<b)?"cn":"global"}'
EOF
  )
  log "region=auto → $REGION(目标机探测)"
fi

# ── 1. k3s HA ────────────────────────────────────────────────────────────────────
# 每台的 --node-ip:private_ip 优先(东西向走 VLAN);NodePort 限私网段;servicelb 关(入口=外部 LB)
k3s_install_env() { # k3s 安装的 region 化(cn 走 rancher 国内镜像)
  if [ "$REGION" = "cn" ]; then printf 'INSTALL_K3S_MIRROR=cn '; fi
}
node_flags() { # node_flags <i> → 该机的 k3s server 通用 flag(字段已在 parser 层严格校验)
  local i="$1" ip cidr flags=""
  ip="$(mvar "$i" PRIVATE_IP)"; [ -z "$ip" ] && ip="$(mvar "$i" HOST)"
  flags="--node-name $(mvar "$i" NAME) --node-ip $ip --advertise-address $ip --disable servicelb"
  # NodePort 限网段:private_cidr 显式优先;IPv4 才推断 /24;FQDN/IPv6 不加(全监听,由外层防火墙管)
  cidr="$(mvar "$i" PRIVATE_CIDR)"
  if [ -z "$cidr" ] && [[ "$ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then cidr="${ip%.*}.0/24"; fi
  [ -n "$cidr" ] && flags="$flags --kube-proxy-arg nodeport-addresses=$cidr"
  printf '%s' "$flags"
}
K3S0_IP="$(mvar 0 PRIVATE_IP)"; [ -z "$K3S0_IP" ] && K3S0_IP="$(mvar 0 HOST)"

if rsudo 0 <<<'test -f /usr/local/bin/k3s && systemctl is-active --quiet k3s' 2>/dev/null; then
  log "1/8 k3s 首节点已在跑,跳过安装"
else
  log "1/8 k3s 首节点($(mvar 0 NAME) @ $(mvar 0 HOST))"
  rsudo 0 <<EOF
set -e
curl -sfL https://get.k3s.io | $(k3s_install_env)INSTALL_K3S_CHANNEL=$K3S_CHANNEL \
  INSTALL_K3S_EXEC="server --cluster-init $(node_flags 0)" sh -
EOF
fi
TOKEN="$(rsudo 0 <<<'cat /var/lib/rancher/k3s/server/node-token')" || die "读不到 k3s node-token"
for ((i=1; i<MACHINE_COUNT; i++)); do
  if rsudo "$i" <<<'test -f /usr/local/bin/k3s && systemctl is-active --quiet k3s' 2>/dev/null; then
    # 活着 ≠ 属于本集群:被复用的机器可能挂在别的集群上,静默跳过=幽灵成员
    if rsudo 0 <<<"k3s kubectl get node $(mvar "$i" NAME) --no-headers >/dev/null 2>&1"; then
      log "1/8 节点 $(mvar "$i" NAME) 已是本集群成员,跳过"
      continue
    fi
    die "机器 $(mvar "$i" HOST) 上 k3s 在跑但不是本集群成员($(mvar "$i" NAME) 不在)——先在该机跑 k3s-uninstall.sh 再重来"
  fi
  log "1/8 加入节点 $(mvar "$i" NAME) @ $(mvar "$i" HOST)"
  rsudo "$i" <<EOF
set -e
curl -sfL https://get.k3s.io | $(k3s_install_env)INSTALL_K3S_CHANNEL=$K3S_CHANNEL \
  K3S_TOKEN='$TOKEN' INSTALL_K3S_EXEC="server --server https://$K3S0_IP:6443 $(node_flags "$i")" sh -
EOF
done
# Ready 等待(embedded etcd 首次选主可能要 ~1min)
rsudo 0 <<EOF || die "k3s 节点未全 Ready"
set -e
for t in \$(seq 1 60); do
  n=\$(k3s kubectl get nodes --no-headers 2>/dev/null | grep -cw Ready || true)
  [ "\$n" -ge "$MACHINE_COUNT" ] && { k3s kubectl get nodes -o wide; exit 0; }
  sleep 5
done
k3s kubectl get nodes -o wide; exit 1
EOF

# ── 2. VXLAN 守卫(有公网口的机器:flannel 8472 只许私网,公网口 DROP;systemd 常驻,
#      k3s 重装会重置 iptables——unit 常驻保证 reboot/重装后规则还在)────────────────
log "2/8 VXLAN 公网口守卫"
for ((i=0; i<MACHINE_COUNT; i++)); do
  iface="$(mvar "$i" PUBLIC_IFACE)"; [ -z "$iface" ] && continue
  rsudo "$i" <<EOF
set -e
cat > /etc/systemd/system/sema-vxlan-guard.service <<'UNIT'
[Unit]
Description=Drop flannel VXLAN on public iface (sema isolation invariant)
After=network-online.target k3s.service
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/sh -c 'iptables -C INPUT -i $iface -p udp --dport 8472 -j DROP 2>/dev/null || iptables -I INPUT -i $iface -p udp --dport 8472 -j DROP'
[Install]
WantedBy=multi-user.target
UNIT
systemctl daemon-reload && systemctl enable --now sema-vxlan-guard.service
iptables -C INPUT -i $iface -p udp --dport 8472 -j DROP
EOF
  log "  $(mvar "$i" NAME):8472/udp DROP on $iface ✓"
done

# ── 3. helm(装配工具,仅首节点)────────────────────────────────────────────────
log "3/8 helm + 基础组件"
rsudo 0 <<'EOF'
set -e
command -v helm >/dev/null && exit 0
curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash >/dev/null
EOF
KC="--kubeconfig /etc/rancher/k3s/k3s.yaml"

# Longhorn(分布式存储;replica 数=min(机器数,2)——RAID1 之上跨机冗余)
LH_REPLICAS=$(( MACHINE_COUNT < 2 ? 1 : 2 ))
rsudo 0 <<EOF
set -e
if ! k3s kubectl get storageclass longhorn >/dev/null 2>&1; then
  helm $KC repo add longhorn https://charts.longhorn.io >/dev/null
  helm $KC repo update >/dev/null
  # ⚠ defaultSettings.* 必须 --set-string(chart 模板校验字符串型,int 直接 install 失败——真踩);
  # upgrade --install:半途失败的 release 重跑可修复(裸 install 会撞已存在)
  helm $KC upgrade --install longhorn longhorn/longhorn --namespace longhorn-system --create-namespace \
    --version $LONGHORN_VERSION --set-string defaultSettings.defaultReplicaCount=$LH_REPLICAS \
    --set persistence.defaultClassReplicaCount=$LH_REPLICAS --wait --timeout 10m >/dev/null
fi
# longhorn=默认 SC,local-path 摘掉 default 标(两个 default 会让 PVC 行为不定)
k3s kubectl patch storageclass local-path -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}' >/dev/null 2>&1 || true
k3s kubectl patch storageclass longhorn -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' >/dev/null
echo LONGHORN_OK
EOF

# kata-deploy(官方 helm chart;kustomize remote base 老路径已失效——真踩)
log "4/8 kata-deploy(VM 沙箱运行时)"
rsudo 0 <<EOF
set -e
k3s kubectl get runtimeclass kata-qemu >/dev/null 2>&1 && { echo KATA_ALREADY; exit 0; }
helm $KC upgrade --install kata-deploy oci://ghcr.io/kata-containers/kata-deploy-charts/kata-deploy \
  --version $KATA_VERSION --namespace kube-system --set k8sDistribution=k3s --wait --timeout 10m >/dev/null
for t in \$(seq 1 36); do k3s kubectl get runtimeclass kata-qemu >/dev/null 2>&1 && break; sleep 5; done
k3s kubectl get runtimeclass kata-qemu >/dev/null
echo KATA_OK
EOF

# CNPG operator(钉 $CNPG_VERSION)
log "5/8 CNPG operator $CNPG_VERSION"
rsudo 0 <<EOF
set -e
if ! k3s kubectl get deployment -n cnpg-system cnpg-controller-manager >/dev/null 2>&1; then
  # 组件刚装完 apiserver 聚合 openapi 可能未就绪(瞬时 "unable to handle the request")——重试非降级
  ok=0
  for t in 1 2 3 4 5; do
    k3s kubectl apply --server-side -f \
      "https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${CNPG_VERSION%.*}/releases/cnpg-$CNPG_VERSION.yaml" >/dev/null && { ok=1; break; }
    sleep 15
  done
  [ "\$ok" = 1 ] || { echo "CNPG manifest apply 重试耗尽" >&2; exit 1; }
fi
k3s kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=300s >/dev/null
echo CNPG_OK
EOF

# ── 6. server 镜像:registry 引用 or tar 预灌 ─────────────────────────────────────
log "6/8 server 镜像"
SERVER_IMAGE="" PULL_POLICY=""
if [ -n "${SEMA_SERVER_IMAGE:-}" ]; then
  SERVER_IMAGE="$SEMA_SERVER_IMAGE"; PULL_POLICY="IfNotPresent"
  log "  registry 引用:$SERVER_IMAGE(节点自行拉取)"
elif [ -n "${SEMA_SERVER_IMAGE_TAR:-}" ]; then
  [ -r "$SEMA_SERVER_IMAGE_TAR" ] || die "SEMA_SERVER_IMAGE_TAR 不可读:$SEMA_SERVER_IMAGE_TAR"
  SERVER_IMAGE="docker.io/library/sema-server:local"; PULL_POLICY="Never"
  for ((i=0; i<MACHINE_COUNT; i++)); do
    # ⚠ 每台都灌,包括"看起来已有"的——k3s-uninstall 重装过的节点镜像会被清(复审真抓)
    log "  预灌 $(mvar "$i" NAME) …"
    mscp "$i" "$SEMA_SERVER_IMAGE_TAR" /tmp/sema-server-image.tar.upload
    rsudo "$i" <<'EOF'
set -e
trap 'rm -f /tmp/sema-server-image.tar.upload /tmp/sema-server-image.tar' EXIT
f=/tmp/sema-server-image.tar.upload
case "$(head -c2 "$f" | od -An -tx1 | tr -d ' ')" in 1f8b) gunzip -c "$f" > /tmp/sema-server-image.tar && f=/tmp/sema-server-image.tar ;; esac
k3s ctr images import "$f" >/dev/null
k3s ctr images ls | grep -q sema-server
EOF
  done
else
  die "需要 SEMA_SERVER_IMAGE(registry 引用)或 SEMA_SERVER_IMAGE_TAR(docker save 包)之一"
fi

# ── 6b. 私有 registry 拉取凭据(可选。SWR 已弃用 2026-07-13:现行镜像池=docker.io/claybobby 公开仓、
#     sema-server=ghcr.io/sema-agent/sema-server,均 public 匿名可拉,CN 兜底=内网 Gitea registry——
#     公开池部署本步冗余无害,regcred 仅在镜像换成 private 仓时才需要;未设 env 即整段跳过)──
# 凭据只走 env:IMAGE_PULL_USER + IMAGE_PULL_TOKEN(+可选 IMAGE_PULL_REGISTRY,默认 Docker Hub)。
# 值不进仓、不进 argv(/proc cmdline 同机可见——渲染同款纪律):dockerconfigjson 本地拼好经 stdin apply。
# secret 建在 sema + sema-sandbox 两个 ns 并 patch default SA 的 imagePullSecrets:
#   - sema:server Deployment(default SA)拉 private 仓镜像(现行 GHCR/Hub 公开镜像无需凭据);
#   - sema-sandbox:server 经 API 起的沙箱 pod(K8S_SANDBOX_IMAGE 换成 private 池镜像时)免改 adapter。
# ⚠ SA 的 imagePullSecrets 在 pod 创建时快照——先建 secret/patch SA,再 apply 栈(顺序有意)。
if [ -n "${IMAGE_PULL_USER:-}" ] && [ -n "${IMAGE_PULL_TOKEN:-}" ]; then
  log "6b/8 imagePullSecret(registry=${IMAGE_PULL_REGISTRY:-https://index.docker.io/v1/})"
  for ns in sema sema-sandbox; do
    rrun 0 k3s kubectl create namespace "$ns" --dry-run=client -o yaml | rrun 0 k3s kubectl apply -f - >/dev/null
    # helm adoption:chart 声明 Namespace/sema-sandbox——6b 先建了它,不打 helm 属主标记
    # 的话 `helm upgrade --install` 拒绝接管(fresh helm 装直接失败)。render 路径不受影响(仅 helm 时打)。
    if [ "$ns" = sema-sandbox ] && [ "${STACK_ENGINE:-render}" = "helm" ]; then
      rrun 0 k3s kubectl label ns "$ns" app.kubernetes.io/managed-by=Helm --overwrite >/dev/null
      rrun 0 k3s kubectl annotate ns "$ns" meta.helm.sh/release-name=sema-stack meta.helm.sh/release-namespace=sema --overwrite >/dev/null
    fi
    IPS_NS="$ns" IPS_REG="${IMAGE_PULL_REGISTRY:-https://index.docker.io/v1/}" \
    IPS_USER="$IMAGE_PULL_USER" IPS_TOKEN="$IMAGE_PULL_TOKEN" python3 - <<'PY' | rrun 0 k3s kubectl apply -f - >/dev/null
import base64, json, os
u, p = os.environ["IPS_USER"], os.environ["IPS_TOKEN"]
cfg = {"auths": {os.environ["IPS_REG"]: {"username": u, "password": p,
       "auth": base64.b64encode(f"{u}:{p}".encode()).decode()}}}
b64 = base64.b64encode(json.dumps(cfg).encode()).decode()
print(json.dumps({"apiVersion": "v1", "kind": "Secret", "type": "kubernetes.io/dockerconfigjson",
      "metadata": {"name": "sema-regcred", "namespace": os.environ["IPS_NS"]},
      "data": {".dockerconfigjson": b64}}))
PY
    # default SA 由 ns 控制器异步建——等它出现再 patch(幂等)
    rsudo 0 <<EOF
set -e
for t in \$(seq 1 12); do k3s kubectl get sa default -n $ns >/dev/null 2>&1 && break; sleep 5; done
k3s kubectl patch sa default -n $ns -p '{"imagePullSecrets":[{"name":"sema-regcred"}]}' >/dev/null
EOF
  done
elif [ -n "${IMAGE_PULL_USER:-}${IMAGE_PULL_TOKEN:-}" ]; then
  die "IMAGE_PULL_USER/IMAGE_PULL_TOKEN 只给了一个——两个都要(或都不给=公共镜像)"
fi

# ── 7. sema-stack 装配:helm chart(STACK_ENGINE=helm)或渲染 apply(默认,as-built)──
# chart/ 与 stack/sema-stack-cluster.yaml 同构(对象名一致,可从存量 adoption);helm 路径
# 验收等价后再翻默认。
log "7/8 sema-stack 渲染+apply(engine=${STACK_ENGINE:-render})"
: "${MODEL_GATEWAY_BASEURL:?模型面必需(--gateway)}"; : "${MODEL_ID:?模型面必需(--model)}"
ENVF="$WORK/.env"; touch "$ENVF"; chmod 600 "$ENVF"
setk() { grep -q "^$1=" "$ENVF" || printf '%s=%s\n' "$1" "$2" >> "$ENVF"; }
setk MINIO_PASSWORD "$(openssl rand -hex 24)"
setk SERVICE_AUTH_TOKEN "$(openssl rand -hex 24)"
# openssl 失败会被 $() 吞成空串写进 .env(set -e 不管参数位替换)——断言长度,fail-loud
for k in MINIO_PASSWORD SERVICE_AUTH_TOKEN; do
  grep -q "^$k=.\{32,\}$" "$ENVF" || die "$k 生成失败(openssl rand?)——检查 $ENVF"
done
. "$ENVF"
[ -n "${MODEL_API_KEY:-}" ] || die "需要 MODEL_API_KEY(-f 配置或 --model-key)"
# PG 实例数/反亲和/副本数按机器数缩放(<3 机=dev:preferred,required 会永久 Pending)
PG_INSTANCES=$(( MACHINE_COUNT >= 3 ? 3 : MACHINE_COUNT ))
PG_AA=$([ "$MACHINE_COUNT" -ge 3 ] && echo required || echo preferred)
SERVER_REPLICAS=$(( MACHINE_COUNT >= 3 ? 3 : 2 ))
if [ "${STACK_ENGINE:-render}" = "helm" ]; then
  # ── helm 路径:chart tar 经 stdin 上载(不落敏感值;chart 本身无凭据),values 本地生成
  #    (umask 077)经 stdin 给 helm -f -(凭据不进 argv、不落远端盘——与渲染路径同纪律)──
  tar -C "$HERE" -czf - chart | mssh 0 'rm -rf /tmp/sema-stack-chart && mkdir -p /tmp/sema-stack-chart && tar xzf - -C /tmp/sema-stack-chart'
  VALSF="$WORK/sema-stack-values.yaml"
  ( umask 077
    V_MINIO_PW="$MINIO_PASSWORD" V_SVC_TOKEN="$SERVICE_AUTH_TOKEN" V_MODEL_KEY="$MODEL_API_KEY" \
    V_MODEL_GATEWAY="$MODEL_GATEWAY_BASEURL" V_MODEL_ID="$MODEL_ID" \
    V_SERVER_IMAGE="$SERVER_IMAGE" V_PULL_POLICY="$PULL_POLICY" \
    V_PKG_SOURCE="$([ "$REGION" = "global" ] && echo global || echo '')" \
    V_PG_INSTANCES="$PG_INSTANCES" V_PG_AA="$PG_AA" V_REPLICAS="$SERVER_REPLICAS" \
    python3 - > "$VALSF" <<'PY'
import json, os
e = os.environ
# json 是合法 yaml;json.dumps 对任意凭据字符安全(引号/反斜杠/换行都正确转义)
print(json.dumps({
  "pg": {"instances": int(e["V_PG_INSTANCES"]), "antiAffinity": e["V_PG_AA"]},
  "minio": {"rootPassword": e["V_MINIO_PW"]},
  "server": {
    "image": e["V_SERVER_IMAGE"], "pullPolicy": e["V_PULL_POLICY"],
    "replicas": int(e["V_REPLICAS"]), "serviceAuthToken": e["V_SVC_TOKEN"],
    "modelGatewayBaseurl": e["V_MODEL_GATEWAY"], "modelId": e["V_MODEL_ID"],
    "modelApiKey": e["V_MODEL_KEY"], "sandboxPkgSource": e["V_PKG_SOURCE"],
  },
  # chart 0.3.0 起 registry.enabled 默认 true(单机档语义)——machines 多机路径的 registry/git
  # 面还没接线,必须显式关,不然 helm 撞 registry.image required(2026-07-12 自查)
  "registry": {"enabled": False},
  "gitea": {"enabled": False},
}))
PY
  )
  [ -s "$VALSF" ] || die "helm values 生成失败"
  # 失败也删 values 文件(die 在 rm 之前退出会把含 MODEL_API_KEY 的 0600 文件留在 WORK)
  rrun 0 helm --kubeconfig /etc/rancher/k3s/k3s.yaml upgrade --install sema-stack \
    /tmp/sema-stack-chart/chart -n sema --create-namespace -f - < "$VALSF" >/dev/null \
    || { rm -f "$VALSF"; die "helm upgrade --install sema-stack 失败"; }
  rm -f "$VALSF"
  rrun 0 rm -rf /tmp/sema-stack-chart
else
RENDERED="$WORK/sema-stack-cluster.rendered.yaml"
# python 逐字替换而非 sed:①密钥/URL 含 & | \ 时 sed 会注入/静默毁 YAML
# ②值走 env 不走 argv(/proc/*/cmdline 对同机进程可见)。占位符缺值=fail-loud。
( umask 077
  R_MINIO_PW="$MINIO_PASSWORD" R_SVC_TOKEN="$SERVICE_AUTH_TOKEN" R_MODEL_KEY="$MODEL_API_KEY" \
  R_MODEL_GATEWAY="$MODEL_GATEWAY_BASEURL" R_MODEL_ID="$MODEL_ID" \
  R_SERVER_IMAGE="$SERVER_IMAGE" R_PULL_POLICY="$PULL_POLICY" \
  R_PKG_SOURCE="$([ "$REGION" = "global" ] && echo global || echo '')" \
  R_PG_INSTANCES="$PG_INSTANCES" R_PG_AA="$PG_AA" R_REPLICAS="$SERVER_REPLICAS" \
  python3 - "$HERE/stack/sema-stack-cluster.yaml" > "$RENDERED" <<'PY'
import os, sys
MAP = {"__MINIO_PW__":"R_MINIO_PW", "__SVC_TOKEN__":"R_SVC_TOKEN", "__MODEL_KEY__":"R_MODEL_KEY",
       "__MODEL_GATEWAY__":"R_MODEL_GATEWAY", "__MODEL_ID__":"R_MODEL_ID",
       "__SERVER_IMAGE__":"R_SERVER_IMAGE", "__SERVER_PULL_POLICY__":"R_PULL_POLICY",
       "__PKG_SOURCE__":"R_PKG_SOURCE", "__PG_INSTANCES__":"R_PG_INSTANCES",
       "__PG_ANTI_AFFINITY__":"R_PG_AA", "__SERVER_REPLICAS__":"R_REPLICAS"}
t = open(sys.argv[1], encoding="utf-8").read()
for ph, env in MAP.items():
    v = os.environ.get(env)
    if v is None: raise SystemExit(f"渲染缺 {env}({ph})")
    if any(c in v for c in "\n\"\\"): raise SystemExit(f"{env} 含 YAML 危险字符(换行/引号/反斜杠)")
    t = t.replace(ph, v)
import re
left = sorted(set(re.findall(r"__[A-Z_]+__", t)))
if left: raise SystemExit(f"未渲染占位符残留: {left}")
sys.stdout.write(t)
PY
)
[ -s "$RENDERED" ] || die "sema-stack 渲染失败"
# 凭据清单经 stdin 直接 apply,不落远端盘(落 /tmp 失败时会残留)
rrun 0 k3s kubectl apply -f - < "$RENDERED" >/dev/null
fi
rsudo 0 <<'EOF'
set -e
# runner token 跨 ns 复制要赶在 server 起 pod 前(晚了=CreateContainerConfigError 干等——真踩);
# SA token secret 由控制器异步填充,先等它有值
TOK=""
for t in $(seq 1 24); do
  TOK=$(k3s kubectl get secret sema-runner-token -n sema-sandbox -o jsonpath='{.data.token}' 2>/dev/null || true)
  [ -n "$TOK" ] && break; sleep 5
done
[ -n "$TOK" ] || { echo "sema-runner-token 未被控制器填充" >&2; exit 1; }
k3s kubectl create secret generic sema-runner-token-copy -n sema --from-literal=token="$(printf '%s' "$TOK" | base64 -d)" \
  --dry-run=client -o yaml | k3s kubectl apply -f - >/dev/null
# PG healthy(冷启拉镜像+init 可要几分钟)
for t in $(seq 1 120); do
  s=$(k3s kubectl get cluster sema-pg -n sema -o jsonpath='{.status.phase}' 2>/dev/null || true)
  [ "$s" = "Cluster in healthy state" ] && break; sleep 5
done
[ "$s" = "Cluster in healthy state" ] || { k3s kubectl get cluster -n sema; echo "PG 未达 healthy" >&2; exit 1; }
k3s kubectl rollout status deployment/sema-server -n sema --timeout=300s >/dev/null
# MinIO 建桶(备份桶+对象桶)
PW=$(k3s kubectl get secret sema-minio -n sema -o jsonpath='{.data.rootPassword}' | base64 -d)
k3s kubectl exec deploy/sema-minio -n sema -- sh -c \
  "mc alias set local http://localhost:9000 sema '$PW' >/dev/null && mc mb --ignore-existing local/sema-pg-backup local/workspaces local/blobs" >/dev/null
echo STACK_OK
EOF

# ── 8. smoke(节点内经 NodePort;外网入口=LB 在云商侧配,打印指引)────────────────
log "8/8 smoke"
# NodePort 被 nodeport-addresses 钉在节点网段——loopback 不在其内,必须打 node-ip(真踩 curl exit 7)
SMOKE=$(rsudo 0 <<EOF
H=\$(curl -m5 -s -o /dev/null -w '%{http_code}' "http://$K3S0_IP:$NODEPORT/health" || echo 000)
A=\$(curl -m5 -s -o /dev/null -w '%{http_code}' "http://$K3S0_IP:$NODEPORT/v1/runs" || echo 000)
echo "health=\$H auth=\$A"
EOF
)
case "$SMOKE" in *"health=200 auth=401"*) : ;; *) die "smoke 失败:$SMOKE(期望 health=200 auth=401)" ;; esac
log "smoke ✓ $SMOKE"
cat >&2 <<EOF

══════════ cluster-up 完成($MACHINE_COUNT 节点)══════════
 入口   NodePort $NODEPORT(限私网段)。公网入口在云商 LB 上配:
        listener TCP:8090 → pool(每节点 private_ip:$NODEPORT,member 必带 subnet_id——Gcore 真踩)
 数据   CNPG sema-pg($PG_INSTANCES 实例,反亲和 $PG_AA)+ MinIO;WAL 归档+每日基备→ s3://sema-pg-backup/
        (⚠ 若手工毁重建 PG:app secret 会换,必须 kubectl rollout restart deploy/sema-server)
 沙箱   kata-qemu runtimeclass(ns sema-sandbox,SA sema-runner)
 凭据   $ENVF(0600;SERVICE_AUTH_TOKEN=API 门)
 管理   ssh 隧道 6443(kubeconfig 在首节点 /etc/rancher/k3s/k3s.yaml,server 改 127.0.0.1)
═══════════════════════════════════════════════════════════
EOF