Public: anyone can read this; nobody can change it.
rehome: node-restart aftercare (uncordon, exact Ready, wait for the forge) #260
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
rehome-node-aftercare → main 2 commits
2 files
changed +26
−4
bin/lib.sh +14 −3
| @@ -1986,10 +1986,21 @@ front_door_grow() { | ||
| 1986 | 1986 | # Each MCP wait on CI is bounded at 5m; on a loaded host (CI's shared |
| 1987 | 1987 | # runner) a template build can outlive that, so a "CI timed out" answer |
| 1988 | 1988 | # gets ONE more idempotent call (the run keeps going; the retry re-joins it). |
| 1989 | _fd_call() { # $1=tool $2=args — mcp_call with one retry on a CI-timeout answer | |
| 1990 | local _a; for _a in 1 2; do | |
| 1989 | # _fd_call — mcp_call with retries for the two TRANSIENT answer shapes every | |
| 1990 | # front-door tool is idempotent against: a CI-timeout (the run keeps going; | |
| 1991 | # the retry re-joins it) and a transport blip INSIDE the MCP ("fetch failed", | |
| 1992 | # ECONN*, a gateway 5xx — the daughter's forge hiccuped mid-call; run 861). | |
| 1993 | # Anything else is a real refusal and returns immediately. | |
| 1994 | _fd_call() { # $1=tool $2=args | |
| 1995 | local _a; for _a in 1 2 3; do | |
| 1991 | 1996 | out= && return 0 |
| 1992 | printf '%s' "" | grep -q 'CI timed out' && [ "" = 1 ] && { echo " front door: $1 — CI still running after 5m, re-joining it…"; continue; } | |
| 1997 | [ "" = 3 ] && return 1 | |
| 1998 | if printf '%s' "" | grep -q 'CI timed out'; then | |
| 1999 | echo " front door: $1 — CI still running after 5m, re-joining it…"; continue | |
| 2000 | fi | |
| 2001 | if printf '%s' "" | grep -qE 'fetch failed|ECONNREFUSED|ECONNRESET|EAI_AGAIN|socket hang up|502 Bad Gateway|503 Service'; then | |
| 2002 | echo " front door: $1 — transient transport error, retrying in 15s…"; sleep 15; continue | |
| 2003 | fi | |
| 1993 | 2004 | return 1 |
| 1994 | 2005 | done; return 1 |
| 1995 | 2006 | } |
bin/rehome +12 −1
| @@ -207,11 +207,22 @@ case "$CTX" in | ||
| 207 | 207 | echo " node mirror recipe installed for git.$NEW_DOMAIN — restarting node(s) to apply (k3s reads it at start)" |
| 208 | 208 | for _node in $_nodes; do docker restart "$_node" >/dev/null; done |
| 209 | 209 | STEP="wait for nodes Ready after mirror restart" |
| 210 | # A restarted k3d node can come back CORDONED (Ready,SchedulingDisabled | |
| 211 | # — gauntlet run 846: forgejo could not reschedule and the port-forward | |
| 212 | # starved). Uncordon every node, then require the status column to be | |
| 213 | # EXACTLY "Ready" — a substring match reads the cordoned state as fine. | |
| 210 | 214 | for _ in $(seq 1 60); do |
| 211 | _notready=$(kubectl get nodes --no-headers 2>/dev/null | grep -cv ' Ready' || true) | |
| 215 | kubectl get nodes -o name 2>/dev/null | while read -r _n; do | |
| 216 | kubectl uncordon "${_n#node/}" >/dev/null 2>&1 || true | |
| 217 | done | |
| 218 | _notready=$(kubectl get nodes --no-headers 2>/dev/null | awk '$2 != "Ready"' | wc -l | tr -d ' ') | |
| 212 | 219 | [ "${_notready:-1}" = 0 ] && break; sleep 5 |
| 213 | 220 | done |
| 214 | 221 | kubectl get nodes --no-headers 2>/dev/null | sed 's/^/ /' |
| 222 | # Give the forge time to reschedule before dialing it — the restart | |
| 223 | # recycled every pod on the node. | |
| 224 | STEP="wait for forgejo after mirror restart" | |
| 225 | kubectl -n "$FORGEJO_NS" rollout status deploy/"$FORGEJO_RELEASE" --timeout=8m >/dev/null 2>&1 || true | |
| 215 | 226 | fj_pf # the restart killed the port-forward |
| 216 | 227 | NODE_STEP_DONE=1 |
| 217 | 228 | fi |
new above
offline