Public: anyone can read this; nobody can change it.

rehome: node-restart aftercare (uncordon, exact Ready, wait for the forge) #260

Merged opened this view on origin

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

rehome-node-aftercare → main 2 commits

  • Merged. rehome-node-aftercare was merged into main.

Review

Sign in to review this change.
2 files changed +26 −4
bin/lib.sh +14 −3
@@ -1986,10 +1986,21 @@ front_door_grow() {
19861986 # Each MCP wait on CI is bounded at 5m; on a loaded host (CI's shared
19871987 # runner) a template build can outlive that, so a "CI timed out" answer
19881988 # gets ONE more idempotent call (the run keeps going; the retry re-joins it).
1989 _fd_call() { # $1=tool $2=args — mcp_call with one retry on a CI-timeout answer
1990 local _a; for _a in 1 2; do
1989 # _fd_call — mcp_call with retries for the two TRANSIENT answer shapes every
1990 # front-door tool is idempotent against: a CI-timeout (the run keeps going;
1991 # the retry re-joins it) and a transport blip INSIDE the MCP ("fetch failed",
1992 # ECONN*, a gateway 5xx — the daughter's forge hiccuped mid-call; run 861).
1993 # Anything else is a real refusal and returns immediately.
1994 _fd_call() { # $1=tool $2=args
1995 local _a; for _a in 1 2 3; do
19911996 out=$(mcp_call "$1" "$2" 900) && return 0
1992 printf '%s' "$out" | grep -q 'CI timed out' && [ "$_a" = 1 ] && { echo " front door: $1 — CI still running after 5m, re-joining it…"; continue; }
1997 [ "$_a" = 3 ] && return 1
1998 if printf '%s' "$out" | grep -q 'CI timed out'; then
1999 echo " front door: $1 — CI still running after 5m, re-joining it…"; continue
2000 fi
2001 if printf '%s' "$out" | grep -qE 'fetch failed|ECONNREFUSED|ECONNRESET|EAI_AGAIN|socket hang up|502 Bad Gateway|503 Service'; then
2002 echo " front door: $1 — transient transport error, retrying in 15s…"; sleep 15; continue
2003 fi
19932004 return 1
19942005 done; return 1
19952006 }
bin/rehome +12 −1
@@ -207,11 +207,22 @@ case "$CTX" in
207207 echo " node mirror recipe installed for git.$NEW_DOMAIN — restarting node(s) to apply (k3s reads it at start)"
208208 for _node in $_nodes; do docker restart "$_node" >/dev/null; done
209209 STEP="wait for nodes Ready after mirror restart"
210 # A restarted k3d node can come back CORDONED (Ready,SchedulingDisabled
211 # — gauntlet run 846: forgejo could not reschedule and the port-forward
212 # starved). Uncordon every node, then require the status column to be
213 # EXACTLY "Ready" — a substring match reads the cordoned state as fine.
210214 for _ in $(seq 1 60); do
211 _notready=$(kubectl get nodes --no-headers 2>/dev/null | grep -cv ' Ready' || true)
215 kubectl get nodes -o name 2>/dev/null | while read -r _n; do
216 kubectl uncordon "${_n#node/}" >/dev/null 2>&1 || true
217 done
218 _notready=$(kubectl get nodes --no-headers 2>/dev/null | awk '$2 != "Ready"' | wc -l | tr -d ' ')
212219 [ "${_notready:-1}" = 0 ] && break; sleep 5
213220 done
214221 kubectl get nodes --no-headers 2>/dev/null | sed 's/^/ /'
222 # Give the forge time to reschedule before dialing it — the restart
223 # recycled every pod on the node.
224 STEP="wait for forgejo after mirror restart"
225 kubectl -n "$FORGEJO_NS" rollout status deploy/"$FORGEJO_RELEASE" --timeout=8m >/dev/null 2>&1 || true
215226 fj_pf # the restart killed the port-forward
216227 NODE_STEP_DONE=1
217228 fi

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs