Public: anyone can read this; nobody can change it.
workbenchthis machine
azure-gate: fix the aks cert-zone doubling, CF 403 preflight, and PASS-exits-1; root-cause the pkill false-FAIL (#158) #159
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
fix/aks-cert-zone-doubling → main 4 commits
3 files
changed +80
−23
bin/lib.sh +21 −1
| @@ -47,8 +47,28 @@ rewrite_identity_domain() { # $1 = gitops clone, $2 = seed/old domain, $3 = new | ||
| 47 | 47 | # domain (and never the hyphenated CA name). The replacement is a fixed string, |
| 48 | 48 | # so dots in $new need no escaping; we use a sed delimiter unlikely to appear. |
| 49 | 49 | local pat; pat= |
| 50 | # Shield values that are ALREADY the new domain. When $new is a subdomain of | |
| 51 | # $old (gate daughters: gl3.gate.open-platform.sh under open-platform.sh), | |
| 52 | # any correct value written by an earlier step still ends in $old, and the | |
| 53 | # rewrite would prepend the sub-labels AGAIN (trim_gitops_aks 4d wrote | |
| 54 | # hostedZoneName: gl3.gate.open-platform.sh; the sweep made it | |
| 55 | # gl3.gate.gl3.gate.open-platform.sh -> DNS-01 403 -> the public cert never | |
| 56 | # issued and germination stalled at the mcp chart build, 2026-08-19). | |
| 57 | # Sentinel-swap $new out, rewrite $old, swap back — idempotent for every | |
| 58 | # already-correct value. ONLY when $new contains $old: in the reverse shape | |
| 59 | # ($old a subdomain of $new's suffix, e.g. re-germinating a gate seed at its | |
| 60 | # parent domain) the shield would eat the middle of every $old occurrence and | |
| 61 | # turn the whole sweep into a silent no-op, so there we keep the plain sweep | |
| 62 | # (which cannot double — $new has no $old inside it to re-expand). | |
| 63 | local newpat keep='__MITOSIS_KEEP_DAUGHTER_DOMAIN__' | |
| 64 | newpat= | |
| 65 | local -a sedx | |
| 66 | case "" in | |
| 67 | *""*) sedx=(-e "s///g" -e "s///g" -e "s///g") ;; | |
| 68 | *) sedx=(-e "s///g") ;; | |
| 69 | esac | |
| 50 | 70 | grep -rIl --include='*.yaml' "" "/clusters/local" 2>/dev/null | while read -r f; do |
| 51 | sed -i.bak "s///g" "" && rm -f ".bak" | |
| 71 | sed -i.bak "" "" && rm -f ".bak" | |
| 52 | 72 | done |
| 53 | 73 | # Verify the surgical invariant held: the CA tokens must SURVIVE intact. If they |
| 54 | 74 | # were clobbered we abort loudly rather than ship a FailedMount daughter. |
deploy/azure/GATE-NOTES.md +36 −15
| @@ -111,19 +111,29 @@ building the mcp chart** (retried 1/4…4/4). Root cause, diagnosed live: | ||
| 111 | 111 | **It is NOT the gate.** `gate.sh` sets `DOMAIN=gl3.gate.open-platform.sh` once |
| 112 | 112 | (line 268) and never mutates it; the deploy log, the Cloudflare delegation, |
| 113 | 113 | and the Azure zone name are all the correct single domain. `bin/germinate` |
| 114 | sets `AKS_DNS_ZONE:=$DOMAIN` (default-only, no transform). So the doubling is | |
| 115 | introduced in the **aks cert-manager templating** — the `trim_gitops_aks` | |
| 116 | azureDNS swap (`bin/lib.sh:~468-490`) writes `hostedZoneName: {AKS_DNS_ZONE}`, | |
| 117 | but the deployed value is doubled, so either the recipe's | |
| 118 | `letsencrypt/clusterissuer.yaml` carries a `<sub>.__DOMAIN__` placeholder that | |
| 119 | germinate's `__DOMAIN__` pass then expands, or the swap composes the zone from | |
| 120 | the wrong parts. **Fix location: `bin/lib.sh` `trim_gitops_aks` 4d + the seed's | |
| 121 | `letsencrypt/clusterissuer.yaml`.** Reproduce: `deploy/azure/gate.sh --mode aks | |
| 122 | --skip-e2e` and `kubectl get clusterissuer -o yaml | grep hostedZoneName`. | |
| 114 | sets `AKS_DNS_ZONE:=$DOMAIN` (default-only, no transform). | |
| 123 | 115 | |
| 124 | Ben's earlier `aks.benjivers.com` platform issued its public cert fine — worth | |
| 125 | checking why that domain escaped the doubling (manual `AKS_DNS_ZONE`? different | |
| 126 | zone shape?) as a clue to the exact template bug. | |
| 116 | **ROOT-CAUSED + FIXED (2026-08-19, Ben).** The trim 4d swap was innocent — it | |
| 117 | writes the correct single zone. The doubling happens LATER, in | |
| 118 | `rewrite_identity_domain` (bin/lib.sh): germinate's identity rewrite maps the | |
| 119 | seed's mother domain (`open-platform.sh`) to the daughter (`$DOMAIN`) with a | |
| 120 | global sed, and a gate daughter's domain ENDS WITH the mother domain — so the | |
| 121 | already-correct `hostedZoneName: gl3.gate.open-platform.sh` matched on its | |
| 122 | suffix and became `gl3.gate.gl3.gate.open-platform.sh`. Fix: the rewrite now | |
| 123 | sentinel-shields values already equal to the daughter domain before the sweep | |
| 124 | (only in that suffix shape; the reverse shape keeps the plain sweep). This is | |
| 125 | gate-specific by construction: only a daughter whose domain ENDS WITH the | |
| 126 | seed's mother domain can hit the sweep, which is why no earlier aks platform | |
| 127 | (different apex, no mother-domain suffix) ever saw it. Reproduce/verify: | |
| 128 | `deploy/azure/gate.sh --mode aks --skip-e2e` and | |
| 129 | `kubectl get clusterissuer -o yaml | grep hostedZoneName`. | |
| 130 | ||
| 131 | Separate CI-only failure (first aks run via Actions, `r420a1`): the | |
| 132 | `AZURE_GATE_CF_TOKEN` repo secret got `403` from Cloudflare on the zone lookup | |
| 133 | — the secret needs an active token with Zone:Read + DNS:Edit on | |
| 134 | `open-platform.sh` (the local `cf-token` file the `l3` run used was fine). | |
| 135 | `cf_load` now reports the HTTP status instead of a json traceback, and the aks | |
| 136 | preflight (token/seed/kubectl) runs BEFORE the RG is created. | |
| 127 | 137 | |
| 128 | 138 | ## State (2026-08-19) |
| 129 | 139 | |
| @@ -138,12 +148,23 @@ zone shape?) as a clue to the exact template bug. | ||
| 138 | 148 | `coldstart-assert.sh`. shellcheck-clean, k3s path byte-identical. |
| 139 | 149 | - ✅ MERGED to main (#143, squash `ebb0864`) — `azure-gate` now shows in the |
| 140 | 150 | Forgejo Actions Run-workflow UI. vms is production-ready. |
| 151 | - ✅ vms false-FAIL after a full PASS (`r425a1`, which already ran #156's | |
| 152 | "specific" pattern): `pkill -f` regex-matches the pattern's own literal text | |
| 153 | inside the carrier shell's argv, so it still killed its own shell and | |
| 154 | dropped the SSH channel. Root cause proven in a Linux container (the #156 | |
| 155 | pattern kills its own carrier, rc=143; a self-match-broken pattern doesn't | |
| 156 | and still matches the real kubectl argv — note macOS pgrep does NOT | |
| 157 | reproduce this, test on Linux). Fixed by #158: `[f]`-bracketed reap pattern | |
| 158 | + the gate result read from the printed `GATE_RC`, not ssh's exit code, with | |
| 159 | a timeout backstop. `r436a1` is its first live exercise. | |
| 141 | 160 | - ✅ aks **harness** proven end to end on `l3`: deploy → CF delegation → |
| 142 | 161 | kubeconfig → ACR mirror → `az acr build` ci-runner → germinate start → |
| 143 | 162 | LB-IP watcher wrote the edge records → clean teardown (RG + delegation). |
| 144 | - ⛔ aks **germination** blocked by the doubled-`hostedZoneName` cert bug above | |
| 145 | (Ben). Until fixed, an aks run stalls at the mcp chart build and eventually | |
| 146 | tears itself down. | |
| 163 | - ✅ doubled-`hostedZoneName` cert bug root-caused + fixed in | |
| 164 | `rewrite_identity_domain` (see above) — proven in a sandbox repro (old code | |
| 165 | doubles, new code doesn't; non-suffix domain shapes unchanged); not yet proven on a | |
| 166 | live aks run. ⬜ CI aks runs additionally need the `AZURE_GATE_CF_TOKEN` | |
| 167 | secret fixed (403 on `r420a1`; Zone:Read + DNS:Edit on `open-platform.sh`). | |
| 147 | 168 | - ⬜ App-E2E (E3) unproven on a fresh cloud platform: on `l2` (vms) the |
| 148 | 169 | dispatcher didn't open the canary's prime PR within 900s. Needs a live |
| 149 | 170 | dispatcher inspection (webhook fired? crew token valid? cold-build timing?). |
deploy/azure/gate.sh +23 −7
| @@ -153,10 +153,17 @@ cf_load() { | ||
| 153 | 153 | elif [ -f "" ]; then CF_TOKEN= |
| 154 | 154 | else died "aks mode needs CF_API_TOKEN or a token file at (delegates )"; fi |
| 155 | 155 | local apex="" # gate.open-platform.sh -> open-platform.sh |
| 156 | CF_ZONE_ID= | |
| 157 | "https://api.cloudflare.com/client/v4/zones?name=" \ | |
| 156 | # Fail with the HTTP status, not a JSONDecodeError traceback: a bad token 403s | |
| 157 | # here, curl -f eats the body, and json.load choked on empty stdin (run r420a1). | |
| 158 | local resp code | |
| 159 | resp= | |
| 160 | "https://api.cloudflare.com/client/v4/zones?name=") \ | |
| 161 | || died "Cloudflare API unreachable (zone lookup for )" | |
| 162 | code= | |
| 163 | [ "" = 200 ] || died "Cloudflare zone lookup for failed (HTTP ) — the token (CI secret AZURE_GATE_CF_TOKEN) must be active with Zone:Read + DNS:Edit on " | |
| 164 | CF_ZONE_ID= | |
| 158 | 165 | | python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r[0]["id"] if r else "")') |
| 159 | [ -n "" ] || died "Cloudflare zone for not found with this token" | |
| 166 | [ -n "" ] || died "Cloudflare zone for not found with this token (Zone:Read scoped to the wrong zone?)" | |
| 160 | 167 | } |
| 161 | 168 | cf() { curl -fsS -H "Authorization: Bearer " -H "Content-Type: application/json" "$@"; } |
| 162 | 169 | cf_delegate() { # $1=fqdn, remaining args = azure NS hosts |
| @@ -218,7 +225,9 @@ DOWN_DONE=0 | ||
| 218 | 225 | down() { |
| 219 | 226 | [ "" = 1 ] && return 0 |
| 220 | 227 | DOWN_DONE=1 |
| 221 | [ "" = aks ] && [ -n "" ] && cf_undelegate "" || true | |
| 228 | # ${DOMAIN:-}: the aks preflight sets CF_ZONE_ID before DOMAIN exists — a | |
| 229 | # death in that window (az group create) must not unbound-crash this trap. | |
| 230 | [ "" = aks ] && [ -n "" ] && [ -n "" ] && cf_undelegate "" || true | |
| 222 | 231 | say "down: deleting resource group " |
| 223 | 232 | az group delete -n "" --yes --no-wait 2>/dev/null || true |
| 224 | 233 | for _ in ; do |
| @@ -246,6 +255,14 @@ on_exit() { | ||
| 246 | 255 | } |
| 247 | 256 | trap on_exit EXIT |
| 248 | 257 | |
| 258 | # ── aks preflight: fail on the cheap stuff BEFORE the RG costs money ───────── | |
| 259 | # (run r420a1 created + deleted an RG just to learn the CF token was bad) | |
| 260 | if [ "" = aks ]; then | |
| 261 | [ -f "/genesis/seed.tar.gz" ] || died "aks mode needs a seed at genesis/seed.tar.gz" | |
| 262 | command -v kubectl >/dev/null || died "aks mode needs kubectl on PATH" | |
| 263 | cf_load | |
| 264 | fi | |
| 265 | ||
| 249 | 266 | # ── RG create (shared) ─────────────────────────────────────────────────────── |
| 250 | 267 | EXPIRES= |
| 251 | 268 | || date -u -v+""H +%Y-%m-%dT%H:%M:%SZ) |
| @@ -328,9 +345,7 @@ if [ "$MODE" = vms ]; then | ||
| 328 | 345 | |
| 329 | 346 | # ═══════════════════════════════ AKS GROUND ═════════════════════════════════ |
| 330 | 347 | else |
| 331 | [ -f "/genesis/seed.tar.gz" ] || died "aks mode needs a seed at genesis/seed.tar.gz" | |
| 332 | command -v kubectl >/dev/null || died "aks mode needs kubectl on PATH" | |
| 333 | cf_load | |
| 348 | # seed/kubectl/cf_load already checked in the aks preflight above | |
| 334 | 349 | DOMAIN="." |
| 335 | 350 | |
| 336 | 351 | say "up: deploying main.bicep (mode=aks, domain=) — a few minutes" |
| @@ -402,3 +417,4 @@ fi | ||
| 402 | 417 | E2E_NOTE=""; { [ "" = 1 ] || [ -z "" ]; } || E2E_NOTE=", app E2E green" |
| 403 | 418 | say "RESULT: PASS () — https://git. germinated, asserted" |
| 404 | 419 | [ "" = 1 ] && say "kept: RG · domain · TTL (card is on the platform, not printed — CI logs are public)" |
| 420 | exit 0 # the && above returns 1 when keep=false — that status would reach the EXIT trap as rc and fail a PASSING run (first seen r436a1, the first CI run to ever reach here green) |
new above
offline