Public: anyone can read this; nobody can change it.

workbenchthis machine

azure-gate: real-cloud CI gate on Azure (vms + aks), manual-dispatch #143

Merged opened this view on origin

Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.

Merged

feat/azure-gate → main 9 commits

  • Merged. feat/azure-gate was merged into main.

Review

Sign in to review this change.
6 files changed +720 −15
.forgejo/workflows/azure-gate.yml added +134
@@ -0,0 +1,134 @@
1 # azure-gate — deploy a REAL mitosis platform on Azure (deploy/azure/main.bicep,
2 # mode=vms, domain=auto → zero DNS), run the cold-start gate + app E2E against
3 # it over SSH, then destroy the resource group. ~$0.30-0.60 per run; per-run
4 # spend is visible in Azure Cost Analysis grouped by tag `run`.
5 #
6 # Manual only (workflow_dispatch): each run costs real money and needs the
7 # AZURE_GATE_* repo secrets, so no pull_request trigger. Inputs:
8 # keep=true leave the platform running (a real deployment you can use);
9 # pass ssh_pubkey so YOU can reach it — the gate's own key is
10 # ephemeral. The janitor reaps it after the TTL (default 5h).
11 # e2e=false skip the app-E2E phase (deploy + assert only)
12 # ref defaults to the branch you dispatch from
13 #
14 # The nightly janitor job deletes any mitosis-gate-* RG whose tags.expires has
15 # passed — leaked platforms (crashed runs, forgotten keeps) die within a day.
16 #
17 # All the logic lives in deploy/azure/gate.sh — run it from a laptop with an
18 # az login and you get the identical cycle. Logs: every phase prints
19 # `==> gate:` markers and the VM's germination journal (`==>` steps) streams
20 # live into the job log.
21 #
22 # Secrets: AZURE_GATE_{CLIENT_ID,CLIENT_SECRET,TENANT_ID,SUBSCRIPTION_ID}
23 # (service principal `mitosis-azure-gate`, Contributor on the sub),
24 # AZURE_GATE_CLAUDE_TOKEN (crew credential sealed at germination —
25 # absent = app E2E is a loud SKIP, deploy+assert still gate).
26
27 name: azure-gate
28 on:
29 workflow_dispatch:
30 inputs:
31 mode:
32 description: ground to test — vms (bare VM, ~$0.30) or aks (managed, ~$1-2)
33 required: false
34 default: vms
35 type: choice
36 options: [vms, aks]
37 keep:
38 description: keep the platform after the gate (true/false)
39 required: false
40 default: "false"
41 e2e:
42 description: run the app E2E phase (true/false)
43 required: false
44 default: "true"
45 ssh_pubkey:
46 description: your SSH public key, authorized on the VM (vms + keep=true)
47 required: false
48 default: ""
49 schedule:
50 - cron: "41 5 * * *" # nightly janitor: reap expired mitosis-gate-* RGs
51
52 concurrency:
53 group: azure-gate
54 cancel-in-progress: false # a cancelled deploy would orphan a paid RG; runs queue instead
55
56 jobs:
57 gate:
58 if: github.event_name == 'workflow_dispatch'
59 runs-on: vxrail-coldstart
60 timeout-minutes: 180
61 steps:
62 - name: Clone (host runner has no node for actions/checkout)
63 run: |
64 set -euo pipefail
65 rm -rf src
66 git clone --depth 50 --branch "${{ github.ref_name }}" \
67 https://git.open-platform.sh/${{ github.repository }} src
68 git -C src log -1 --oneline
69
70 - name: Azure gate (deploy → assert → e2e → destroy)
71 env:
72 AZURE_GATE_CLIENT_ID: ${{ secrets.AZURE_GATE_CLIENT_ID }}
73 AZURE_GATE_CLIENT_SECRET: ${{ secrets.AZURE_GATE_CLIENT_SECRET }}
74 AZURE_GATE_TENANT_ID: ${{ secrets.AZURE_GATE_TENANT_ID }}
75 AZURE_GATE_SUBSCRIPTION_ID: ${{ secrets.AZURE_GATE_SUBSCRIPTION_ID }}
76 AZURE_GATE_CLAUDE_TOKEN: ${{ secrets.AZURE_GATE_CLAUDE_TOKEN }}
77 CF_API_TOKEN: ${{ secrets.AZURE_GATE_CF_TOKEN }}
78 GATE_RUN: r${{ github.run_number }}a${{ github.run_attempt }}
79 MODE: ${{ github.event.inputs.mode }}
80 MITOSIS_REF: ${{ github.ref_name }}
81 GATE_SSH_PUBKEY: ${{ github.event.inputs.ssh_pubkey }}
82 # az config lives under the job workspace, not the runner user's home
83 AZURE_CONFIG_DIR: ${{ github.workspace }}/.azure
84 run: |
85 set -euo pipefail
86 ARGS=""
87 [ "${{ github.event.inputs.keep }}" = "true" ] && ARGS="$ARGS --keep"
88 [ "${{ github.event.inputs.e2e }}" = "false" ] && ARGS="$ARGS --skip-e2e"
89 # Orchestrator tools on PATH; gate.sh layers `nix develop` (flake tools)
90 # for germinate + coldstart-assert. az/kubectl/jq/ssh/curl cover both modes.
91 nix shell nixpkgs#azure-cli nixpkgs#kubectl nixpkgs#jq nixpkgs#openssh nixpkgs#curl --command \
92 bash src/deploy/azure/gate.sh $ARGS
93
94 - name: Safety net — the RG must be gone (unless keep)
95 if: always()
96 env:
97 AZURE_GATE_CLIENT_ID: ${{ secrets.AZURE_GATE_CLIENT_ID }}
98 AZURE_GATE_CLIENT_SECRET: ${{ secrets.AZURE_GATE_CLIENT_SECRET }}
99 AZURE_GATE_TENANT_ID: ${{ secrets.AZURE_GATE_TENANT_ID }}
100 AZURE_GATE_SUBSCRIPTION_ID: ${{ secrets.AZURE_GATE_SUBSCRIPTION_ID }}
101 AZURE_CONFIG_DIR: ${{ github.workspace }}/.azure
102 run: |
103 set -euo pipefail
104 [ "${{ github.event.inputs.keep }}" = "true" ] && { echo "keep=true — RG intentionally alive (janitor TTL applies)"; exit 0; }
105 nix shell nixpkgs#azure-cli --command bash -c '
106 az account show >/dev/null 2>&1 || az login --service-principal \
107 -u "$AZURE_GATE_CLIENT_ID" -p "$AZURE_GATE_CLIENT_SECRET" \
108 --tenant "$AZURE_GATE_TENANT_ID" --output none
109 az account set -s "$AZURE_GATE_SUBSCRIPTION_ID"
110 RG="mitosis-gate-r${{ github.run_number }}a${{ github.run_attempt }}"
111 if az group exists -n "$RG" | grep -qi true; then
112 echo "RG $RG still exists — deleting (safety net)"
113 az group delete -n "$RG" --yes --no-wait
114 else
115 echo "RG $RG gone — clean"
116 fi'
117
118 janitor:
119 if: github.event_name == 'schedule'
120 runs-on: vxrail-coldstart
121 timeout-minutes: 20
122 steps:
123 - name: Reap expired mitosis-gate-* resource groups
124 env:
125 AZURE_GATE_CLIENT_ID: ${{ secrets.AZURE_GATE_CLIENT_ID }}
126 AZURE_GATE_CLIENT_SECRET: ${{ secrets.AZURE_GATE_CLIENT_SECRET }}
127 AZURE_GATE_TENANT_ID: ${{ secrets.AZURE_GATE_TENANT_ID }}
128 AZURE_GATE_SUBSCRIPTION_ID: ${{ secrets.AZURE_GATE_SUBSCRIPTION_ID }}
129 AZURE_CONFIG_DIR: ${{ github.workspace }}/.azure
130 run: |
131 set -euo pipefail
132 rm -rf src
133 git clone --depth 1 https://git.open-platform.sh/${{ github.repository }} src
134 nix shell nixpkgs#azure-cli --command bash src/deploy/azure/gate.sh --janitor
deploy/azure/GATE-NOTES.md added +120
@@ -0,0 +1,120 @@
1 # Azure gate — build notes & handoff
2
3 Running log for the `azure-gate` CI (deploy a real mitosis platform on Azure,
4 test it, destroy it). Written so Ben (and his agent) can pick this up. Status
5 markers: ✅ done · 🚧 in progress · ⬜ todo.
6
7 ## What it is
8
9 One workflow `.forgejo/workflows/azure-gate.yml` + one script
10 `deploy/azure/gate.sh` that deploys `deploy/azure/main.bicep`, waits for
11 germination, runs `scripts/coldstart-assert.sh` (+ `--e2e`), then destroys the
12 resource group. Two grounds, chosen per run:
13
14 - **`vms`** (✅ working) — `mode=vms`, `domain=auto` (sslip.io, zero DNS).
15 ~$0.30/run, ~1h. The autonomous smoke test.
16 - **`aks`** (🚧) — `mode=aks`, real sub-domain of `open-platform.sh` delegated
17 to a per-run Azure DNS zone. ~$1–2/run, ~2h. Tests the product ground.
18
19 `gate.sh` runs identically from a laptop (`az login`) and from CI (service
20 principal). All logic in the script; the workflow stays declarative.
21
22 ## Identity & secrets (already provisioned)
23
24 - **Service principal** `mitosis-azure-gate`, appId
25 `e9097ccc-2eb7-4b27-a461-b50baa8ef322`, **Contributor** on subscription
26 `1d655bbf-4fc9-48dd-bf75-2d3f72bb64a1`. Rotate with
27 `az ad sp credential reset --id <appId>`.
28 - **Repo Actions secrets** (plat/mitosis): `AZURE_GATE_CLIENT_ID`,
29 `AZURE_GATE_CLIENT_SECRET`, `AZURE_GATE_TENANT_ID`,
30 `AZURE_GATE_SUBSCRIPTION_ID`, `AZURE_GATE_CLAUDE_TOKEN` (crew credential for
31 the app E2E — absent = E2E is a loud SKIP, deploy+assert still gate).
32 Set them with the `plat` admin PAT (ci-bot lacks the secrets scope):
33 `curl -u plat:<PAT> -X PUT .../repos/plat/mitosis/actions/secrets/<NAME> -d '{"data":"..."}'`.
34 - **Cloudflare token** for `open-platform.sh` lives at repo-root `cf-token`
35 (gitignored — NEVER commit). Zone id `2fbcb5331b09d502067702811494d9be`.
36 Used only by the `aks` path for per-run sub-domain delegation.
37
38 ## DNS design (aks) — proven
39
40 Per run, fully self-cleaning, no standing zone:
41
42 1. `main.bicep mode=aks createDnsZone=true` creates an Azure DNS zone named
43 `<run>.gate.open-platform.sh` and outputs its 4 `dnsNameServers`.
44 2. `gate.sh` adds those as **NS records** under `open-platform.sh` in
45 Cloudflare (delegating `<run>.gate.open-platform.sh` → Azure).
46 3. cert-manager does DNS-01 in the Azure zone (public LE cert). The
47 `letsencrypt.org` CAA on `open-platform.sh` is inherited → wildcard
48 issuance allowed. ✅ verified.
49 4. When the traefik LB gets its external IP, `gate.sh` writes `*` and `@` A
50 records into the Azure zone.
51 5. Teardown deletes the RG (takes the Azure zone with it) **and** removes the
52 Cloudflare NS records.
53
54 **Proof done 2026-08-19**: throwaway zone `dnstest.gate.open-platform.sh`
55 delegated via CF, resolved through Azure on both 1.1.1.1 and 8.8.8.8, torn
56 down clean. The `*.open-platform.sh` proxied tunnel CNAME does NOT shadow it
57 (wildcards match one label only).
58
59 ## Isolation & cleanup (both grounds)
60
61 - Every run = its own RG `mitosis-gate-<run>` (CI: `r<runNumber>a<attempt>`),
62 tagged `purpose=azure-gate run=<> ref=<> expires=<iso>`.
63 - `gate.sh` teardown runs on **any** exit unless `--keep`.
64 - Workflow has an `if: always()` safety-net step that force-deletes the RG.
65 - Nightly **janitor** (`gate.sh --janitor`, cron in the workflow) deletes any
66 `mitosis-gate-*` RG whose `expires` tag has passed — reaps crashed runs and
67 forgotten `--keep`s within a day.
68 - Cost: group Azure Cost Analysis by tag `run` for per-run spend.
69
70 ## Security
71
72 - `ssh_pubkey` workflow input is validated to one well-formed key line and
73 piped over stdin (never interpolated into the remote command) — an
74 adversarial review caught a command-injection → root-RCE here; fixed.
75 - `claudeCodeOauthToken` is `@secure()` in bicep (not in ARM history). The
76 gate VM logs are streamed to the (public) CI log filtered to `==>|FATAL`, so
77 no card/secret content is printed. `--keep` does NOT print the card.
78
79 ## Gotchas found while proving it (2026-08-19)
80
81 - **SSH gate phase hangs after a PASS (vms).** `coldstart-assert` opens a
82 `kubectl port-forward`; its process can orphan (reparent to init) and hold
83 the SSH channel open, so `run_gate` never returns even though the assert
84 exited 0. Fix: the remote command reaps `port-forward svc/forgejo-http` and
85 returns the assert's real rc; SSH keepalive drops a dead channel. Do NOT
86 move the `pkill` into `coldstart-assert.sh` — the k3d coldstart gate runs
87 many in parallel on one host and a broad pkill would kill siblings. The
88 gate VM is single-tenant, so pkill there is safe. (aks is unaffected — its
89 assert runs locally, no SSH channel.)
90 - `pgrep -f GATE_RUN=l3` finds nothing even when alive: `env VAR=x` puts the
91 var in the environment, not argv. Match on `gate.sh --mode aks` instead.
92
93 ## State (2026-08-19)
94
95 - ✅ vms `gate.sh` + workflow, committed on `feat/azure-gate`.
96 - ✅ `coldstart-assert.sh` `NODE_MODE=local` (bare-VM A5; k3d path unchanged).
97 - ✅ Azure SP + 5 repo secrets + cf-token confirmed.
98 - ✅ DNS per-run delegation proven end-to-end.
99 - ✅ Local vms proof run (`l2`, leader 52.188.125.153): germination + the FULL
100 cold-start gate PASSED on real Azure — A1–A12 green, including A5 via
101 `NODE_MODE=local`. app E2E running at time of writing.
102 - ✅ aks path in `gate.sh` (`--mode aks`) + `GROUND=aks` guards in
103 `coldstart-assert.sh` (skip A5; lower Kustomization floor `KUST_MIN`;
104 A9-grafana self-skips). shellcheck-clean, k3s path byte-identical.
105 - 🚧 Local aks proof run (`l3`) in flight — first end-to-end aks exercise;
106 iterating on real failures.
107 - ⬜ Known risk: `aks-image-redirect` mutates the Pod but not the CNPG
108 `Cluster` spec → tenant DBs restart-loop (from `aks-cost-experiment.md`).
109 The app E2E on aks will likely surface this; fix is to extend the Kyverno
110 policy to `clusters.postgresql.cnpg.io` spec.imageName.
111 - ⬜ Open decision (from README truth pass): `mode=aks` bicep defaults still
112 provision the $364/mo baseline (2×D4ds_v6 + Container Insights + managed
113 Prometheus). Fine for a short-lived gate; revisit for the shipped default.
114
115 ## How to run
116
117 - CI: Forgejo → Actions → `azure-gate` → Run workflow. Inputs: `mode` (vms|aks),
118 `keep`, `e2e`, `ssh_pubkey`.
119 - Laptop: `deploy/azure/gate.sh [--mode aks] [--keep] [--skip-e2e]` with an
120 `az login`. For aks it also reads `cf-token` at repo root.
deploy/azure/README.md +60 −8
@@ -396,13 +396,65 @@ NOTE (`aks` mode): the Key Vault has soft delete on. Purge it before you
396396 create a new vault with the same name:
397397 `az keyvault purge -n <keyVaultName>`.
398398
399 ## The Azure CI gate
400
401 `azure-gate` deploys a real mitosis platform on Azure, tests it end to end
402 (germinate → cold-start assertions → build a canary app and follow it to
403 prod), then destroys everything. Each run is its own resource group, so runs
404 never collide and cleanup is one delete.
405
406 ### Run it (Forgejo)
407
408 **Actions → azure-gate → Run workflow.** Choose:
409
410 | Input | Meaning |
411 | ------------ | ------------------------------------------------------------------------ |
412 | `mode` | `vms` (bare VM, ~$0.30, ~1h) or `aks` (managed cluster, ~$1–2, ~2h) |
413 | `e2e` | `true` also builds + tests a canary app; `false` = deploy + assert only |
414 | `keep` | `true` leaves the platform running (a real deployment you can use) |
415 | `ssh_pubkey` | for `vms` + `keep`: paste your `~/.ssh/id_ed25519.pub` so you can log in |
416
417 Watch the live log — each phase prints `==> gate:` and the germination steps
418 stream in. A green run means deployed, tested, and torn down. A `keep` run
419 ends by printing how to reach the platform.
420
421 ### Run it (your laptop)
422
423 `az login`, then from the repo root:
424
425 ```sh
426 deploy/azure/gate.sh # vms, full cycle
427 deploy/azure/gate.sh --mode aks # aks, full cycle
428 deploy/azure/gate.sh --keep # keep the platform running
429 deploy/azure/gate.sh --mode aks --skip-e2e # deploy + assert only
430 ```
431
432 `aks` mode also needs the `cf-token` file at the repo root (delegates a
433 per-run sub-domain of `gate.open-platform.sh`).
434
435 ### Reach or remove a kept platform
436
437 - `vms`: `ssh mitosis@<ip>` (the run prints the IP; logins are in
438 `sudo cat /root/platform-card.txt` on the VM).
439 - `aks`: `az aks get-credentials -g mitosis-gate-<run> -n mitosis-aks`, then
440 `kubectl`.
441 - Remove it: `az group delete -n mitosis-gate-<run> --yes`. If you forget,
442 the nightly janitor deletes any gate resource group whose `expires` tag has
443 passed (default 5h).
444
445 ### Cost
446
447 In Azure **Cost analysis**, group by tag `run` to see each run's spend. A
448 `vms` run is a few tens of cents; an `aks` run is a dollar or two.
449
399450 ## Files in this directory
400451
401 | File | Purpose |
402 | ------------------------- | ----------------------------------------------------------- |
403 | `main.bicep` | The deployment template. Both modes. |
404 | `main.bicepparam.example` | Example parameter file. Copy, edit, do not commit the copy. |
405 | `servers.bicep` | Add two k3s servers to a live `vms` platform. |
406 | `mirror-images.sh` | Copy the infra images into the ACR (`aks` mode). |
407 | `entra-forgejo-oidc.sh` | Add Entra ID sign-in to Forgejo (`aks` mode). |
408 | `aks-cost-experiment.md` | Record of the first AKS platform: costs, commands, lessons. |
452 | File | Purpose |
453 | ------------------------- | ---------------------------------------------------------------------------- |
454 | `main.bicep` | The deployment template. Both modes. |
455 | `main.bicepparam.example` | Example parameter file. Copy, edit, do not commit the copy. |
456 | `servers.bicep` | Add two k3s servers to a live `vms` platform. |
457 | `mirror-images.sh` | Copy the infra images into the ACR (`aks` mode). |
458 | `entra-forgejo-oidc.sh` | Add Entra ID sign-in to Forgejo (`aks` mode). |
459 | `aks-cost-experiment.md` | Record of the first AKS platform: costs, commands, lessons. |
460 | `gate.sh` | The Azure CI gate: deploy, test, destroy. Used by the `azure-gate` workflow. |
deploy/azure/gate.sh added +339
@@ -0,0 +1,339 @@
1 #!/usr/bin/env bash
2 # gate.sh — the Azure cloud gate: deploy a REAL mitosis platform on Azure,
3 # assert it cold-start-healthy, run the app E2E (crew builds a canary app →
4 # preview → merge → prod), then destroy the resource group. One script, two
5 # grounds, runs identically from a laptop and from CI
6 # (.forgejo/workflows/azure-gate.yml).
7 #
8 # deploy/azure/gate.sh # vms ground, full cycle
9 # deploy/azure/gate.sh --mode aks # aks (managed) ground, full cycle
10 # deploy/azure/gate.sh --keep # leave the platform running
11 # deploy/azure/gate.sh --skip-e2e # deploy + assert only
12 # deploy/azure/gate.sh --janitor # reap expired gate RGs, then exit
13 #
14 # Grounds:
15 # vms (default) — main.bicep mode=vms, domain=auto (sslip.io → zero DNS). The
16 # leader VM germinates; the gate watches + asserts over SSH. ~$0.30, ~1h.
17 # aks — main.bicep mode=aks + a per-run Azure DNS zone
18 # <run>.gate.open-platform.sh, delegated in Cloudflare (cf-token at repo
19 # root). The gate germinates locally against the cluster kubeconfig
20 # (CLUSTER_PROFILE=aks), writes the edge A records once the LB IP appears,
21 # asserts with GROUND=aks. ~$1-2, ~2h.
22 #
23 # Auth: current `az` login if present; else the service principal in
24 # AZURE_GATE_CLIENT_ID/_CLIENT_SECRET/_TENANT_ID (+ _SUBSCRIPTION_ID). CI
25 # provides these from repo secrets.
26 #
27 # Env knobs:
28 # MODE / --mode vms | aks (default vms)
29 # GATE_RUN run id → RG mitosis-gate-$GATE_RUN (default: stamp+pid)
30 # MITOSIS_REF branch/tag the platform is built from (default: main)
31 # LOCATION Azure region (default: eastus)
32 # VM_SIZE vms leader size (default Standard_D4as_v7)
33 # GATE_BASE_DOMAIN aks parent domain (default gate.open-platform.sh)
34 # CF_TOKEN_FILE Cloudflare token path (default: <repo>/cf-token)
35 # AZURE_GATE_CLAUDE_TOKEN crew credential sealed at germination; absent = E2E SKIP
36 # GATE_TTL_HOURS expiry tag for the janitor (default 5)
37 #
38 # Every phase prints `==> gate:` markers; the platform's own `==>` germination
39 # log streams live into this log. Resources carry tags (purpose/run/ref/expires)
40 # so Cost Analysis groups per-run spend and the janitor reaps leaks.
41 set -euo pipefail
42
43 HERE=$(cd "$(dirname "$0")" && pwd)
44 REPO=$(cd "$HERE/../.." && pwd)
45 KEEP=0 SKIP_E2E=0 JANITOR=0
46 MODE="${MODE:-vms}"
47 while [ $# -gt 0 ]; do case "$1" in
48 --mode) MODE="$2"; shift 2 ;;
49 --mode=*) MODE="${1#*=}"; shift ;;
50 --vms) MODE=vms; shift ;;
51 --aks) MODE=aks; shift ;;
52 --keep) KEEP=1; shift ;;
53 --skip-e2e) SKIP_E2E=1; shift ;;
54 --janitor) JANITOR=1; shift ;;
55 *) echo "unknown arg: $1" >&2; exit 2 ;;
56 esac; done
57 case "$MODE" in vms|aks) ;; *) echo "MODE must be vms or aks (got: $MODE)" >&2; exit 2 ;; esac
58
59 : "${LOCATION:=eastus}"
60 : "${MITOSIS_REF:=main}"
61 : "${VM_SIZE:=Standard_D4as_v7}"
62 : "${GATE_TTL_HOURS:=5}"
63 : "${GATE_BASE_DOMAIN:=gate.open-platform.sh}"
64 : "${CF_TOKEN_FILE:=$REPO/cf-token}"
65 GATE_RUN="${GATE_RUN:-$(date -u +%m%d%H%M)p$$}"
66 # DNS-label-safe run slug (aks subdomain): lowercase alnum, must start alpha.
67 SLUG=$(printf 'g%s' "$GATE_RUN" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9')
68 RG="mitosis-gate-${GATE_RUN}"
69 T0=$SECONDS
70 say() { echo "==> gate: $*"; }
71 died() { echo "==> gate: FATAL: $*" >&2; exit 1; }
72 # portable timeout (macOS has neither `timeout` nor, by default, `gtimeout`)
73 if command -v timeout >/dev/null 2>&1; then _timeout() { timeout "$@"; }
74 elif command -v gtimeout >/dev/null 2>&1; then _timeout() { gtimeout "$@"; }
75 else _timeout() { shift; "$@"; }; fi # no timeout available: run without a cap
76
77 # ── auth ─────────────────────────────────────────────────────────────────────
78 command -v az >/dev/null || died "az CLI not found on PATH"
79 if ! az account show >/dev/null 2>&1; then
80 : "${AZURE_GATE_CLIENT_ID:?no az login and no service principal in env}"
81 say "logging in as service principal ${AZURE_GATE_CLIENT_ID}"
82 az login --service-principal -u "$AZURE_GATE_CLIENT_ID" \
83 -p "$AZURE_GATE_CLIENT_SECRET" --tenant "$AZURE_GATE_TENANT_ID" --output none
84 fi
85 [ -n "${AZURE_GATE_SUBSCRIPTION_ID:-}" ] && az account set -s "$AZURE_GATE_SUBSCRIPTION_ID"
86 SUB=$(az account show --query id -o tsv)
87 say "mode=$MODE subscription=$SUB region=$LOCATION"
88
89 # ── Cloudflare helpers (aks per-run sub-domain delegation) ───────────────────
90 CF_ZONE_ID="" CF_TOKEN=""
91 cf_load() {
92 # CI passes the token as CF_API_TOKEN (repo secret); a laptop uses the
93 # gitignored repo-root cf-token file.
94 if [ -n "${CF_API_TOKEN:-}" ]; then CF_TOKEN="$CF_API_TOKEN"
95 elif [ -f "$CF_TOKEN_FILE" ]; then CF_TOKEN=$(tr -d '\n' < "$CF_TOKEN_FILE")
96 else died "aks mode needs CF_API_TOKEN or a token file at $CF_TOKEN_FILE (delegates $GATE_BASE_DOMAIN)"; fi
97 local apex="${GATE_BASE_DOMAIN#*.}" # gate.open-platform.sh -> open-platform.sh
98 CF_ZONE_ID=$(curl -fsS -H "Authorization: Bearer $CF_TOKEN" \
99 "https://api.cloudflare.com/client/v4/zones?name=$apex" \
100 | python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r[0]["id"] if r else "")')
101 [ -n "$CF_ZONE_ID" ] || died "Cloudflare zone for $apex not found with this token"
102 }
103 cf() { curl -fsS -H "Authorization: Bearer $CF_TOKEN" -H "Content-Type: application/json" "$@"; }
104 cf_delegate() { # $1=fqdn, remaining args = azure NS hosts
105 local fqdn="$1"; shift
106 local ns
107 for ns in "$@"; do
108 ns=${ns%.}
109 cf -X POST "https://api.cloudflare.com/client/v4/zones/$CF_ZONE_ID/dns_records" \
110 -d "{\"type\":\"NS\",\"name\":\"$fqdn\",\"content\":\"$ns\",\"ttl\":120}" >/dev/null \
111 && say "dns: delegated $fqdn -> $ns" || say "dns: WARNING: NS create failed for $ns"
112 done
113 }
114 cf_undelegate() { # $1=fqdn — remove every NS record we added
115 local fqdn="$1" id
116 for id in $(cf "https://api.cloudflare.com/client/v4/zones/$CF_ZONE_ID/dns_records?type=NS&name=$fqdn" \
117 | python3 -c 'import sys,json;[print(r["id"]) for r in json.load(sys.stdin).get("result",[])]' 2>/dev/null); do
118 cf -X DELETE "https://api.cloudflare.com/client/v4/zones/$CF_ZONE_ID/dns_records/$id" >/dev/null \
119 && say "dns: removed delegation record $id" || true
120 done
121 }
122
123 # ── janitor mode: reap expired gate RGs, then exit ───────────────────────────
124 if [ "$JANITOR" = 1 ]; then
125 NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
126 say "janitor: reaping RGs tagged purpose=azure-gate with expires < $NOW"
127 REAPED=0
128 while IFS=$'\t' read -r name expires basedom; do
129 [ -n "$name" ] || continue
130 case "$name" in mitosis-gate-*) ;; *) say "janitor: $name not mitosis-gate-* — leaving it"; continue ;; esac
131 if [[ "$expires" < "$NOW" ]]; then
132 say "janitor: deleting expired $name (expired $expires)"
133 # remove any Cloudflare delegation this run left behind
134 if [ -n "$basedom" ] && [ "$basedom" != None ]; then
135 cf_load 2>/dev/null && cf_undelegate "$basedom" || true
136 fi
137 az group delete -n "$name" --yes --no-wait
138 REAPED=$((REAPED + 1))
139 else
140 say "janitor: $name alive until $expires — leaving it"
141 fi
142 done < <(az group list --tag purpose=azure-gate --query "[].[name, tags.expires, tags.basedomain]" -o tsv)
143 say "janitor: done ($REAPED deletion(s) started)"
144 exit 0
145 fi
146
147 # ── work dir + ephemeral ssh key (vms) ───────────────────────────────────────
148 WORK=$(mktemp -d)
149 trap 'rm -rf "$WORK"' EXIT # replaced by the full on_exit trap once teardown is defined
150 export KUBECONFIG="$WORK/kubeconfig"
151 SSH_KEY="$WORK/gate-key"
152 ssh-keygen -q -t ed25519 -f "$SSH_KEY" -N "" -C "azure-gate-$GATE_RUN"
153 SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
154 -o ConnectTimeout=10 -o BatchMode=yes -o LogLevel=ERROR
155 -o ServerAliveInterval=20 -o ServerAliveCountMax=6) # drop a truly dead channel
156 vm() { ssh "${SSH_OPTS[@]}" "mitosis@$IP" "$@"; }
157
158 # ── teardown: runs on ANY exit unless --keep ─────────────────────────────────
159 DOWN_DONE=0
160 down() {
161 [ "$DOWN_DONE" = 1 ] && return 0
162 DOWN_DONE=1
163 [ "$MODE" = aks ] && [ -n "$CF_ZONE_ID" ] && cf_undelegate "$DOMAIN" || true
164 say "down: deleting resource group $RG"
165 az group delete -n "$RG" --yes --no-wait 2>/dev/null || true
166 for _ in $(seq 1 60); do
167 az group exists -n "$RG" 2>/dev/null | grep -qi false && { say "down: $RG gone"; return 0; }
168 sleep 15
169 done
170 say "down: WARNING: $RG still deleting after 15m — the janitor will confirm"
171 }
172 on_exit() {
173 local rc=$?
174 [ -n "${TAIL_PID:-}" ] && kill "$TAIL_PID" 2>/dev/null || true
175 [ -n "${GERM_PID:-}" ] && kill "$GERM_PID" 2>/dev/null || true
176 if [ "$KEEP" = 1 ]; then
177 say "keep: platform stays up. Reach it / destroy it:"
178 [ "$MODE" = vms ] && echo " ssh mitosis@${IP:-<ip>} # with the key you passed as ssh_pubkey"
179 [ "$MODE" = aks ] && echo " az aks get-credentials -g $RG -n mitosis-aks # then kubectl"
180 echo " az group delete -n $RG --yes # janitor reaps it after expires=${EXPIRES:-unset} regardless"
181 else
182 down
183 fi
184 rm -rf "$WORK"
185 MIN=$(((SECONDS - T0) / 60))
186 say "total wall clock ${MIN}m"
187 exit $rc
188 }
189 trap on_exit EXIT
190
191 # ── RG create (shared) ───────────────────────────────────────────────────────
192 EXPIRES=$(date -u -d "+${GATE_TTL_HOURS} hours" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \
193 || date -u -v+"${GATE_TTL_HOURS}"H +%Y-%m-%dT%H:%M:%SZ)
194 CREW="${AZURE_GATE_CLAUDE_TOKEN:-}"
195 say "up: creating $RG (mode=$MODE ref=$MITOSIS_REF expires=$EXPIRES)"
196 BASETAG=""; [ "$MODE" = aks ] && BASETAG="$SLUG.$GATE_BASE_DOMAIN"
197 az group create -n "$RG" -l "$LOCATION" --output none \
198 --tags purpose=azure-gate run="$GATE_RUN" ref="$MITOSIS_REF" expires="$EXPIRES" \
199 mode="$MODE" basedomain="${BASETAG:-none}"
200
201 # ═══════════════════════════════ VMS GROUND ═════════════════════════════════
202 if [ "$MODE" = vms ]; then
203 MYIP=$(curl -fsS -m 10 https://api.ipify.org 2>/dev/null || true)
204 SSH_CIDR="${MYIP:+$MYIP/32}"; SSH_CIDR="${SSH_CIDR:-*}"
205 [ "$SSH_CIDR" = "*" ] && say "up: WARNING: could not learn our egress IP — SSH open to * for this run"
206
207 say "up: deploying main.bicep (mode=vms, domain=auto, $VM_SIZE, ssh from $SSH_CIDR)"
208 az deployment group create -g "$RG" -n gate -f "$HERE/main.bicep" --output none \
209 -p domain=auto -p adminSshPublicKey="$(cat "$SSH_KEY.pub")" -p vmSize="$VM_SIZE" \
210 -p allowedSshCidr="$SSH_CIDR" -p mitosisRef="$MITOSIS_REF" -p claudeCodeOauthToken="$CREW"
211 IP=$(az deployment group show -g "$RG" -n gate --query 'properties.outputs.nodeIps.value[0]' -o tsv)
212 [ -n "$IP" ] || died "deployment produced no node IP"
213 DOMAIN="$(echo "$IP" | tr . -).sslip.io"
214 say "up: leader $IP → https://git.$DOMAIN"
215
216 say "germinate: waiting for SSH on $IP"
217 for _ in $(seq 1 40); do if vm true 2>/dev/null; then break; fi; sleep 15; done
218 vm true || died "SSH to $IP never came up"
219 if [ -n "${GATE_SSH_PUBKEY:-}" ]; then
220 if printf '%s' "$GATE_SSH_PUBKEY" | grep -Eq '^(ssh-ed25519|ssh-rsa|ecdsa-sha2-[a-z0-9-]+|[email protected]|sk-ecdsa-sha2-[a-z0-9-][email protected]) [A-Za-z0-9+/=]+( [^\r\n]*)?$' \
221 && [ "$(printf '%s' "$GATE_SSH_PUBKEY" | wc -l)" -eq 0 ]; then
222 say "germinate: authorizing the operator's SSH key on the VM (for --keep)"
223 printf '%s\n' "$GATE_SSH_PUBKEY" | ssh "${SSH_OPTS[@]}" "mitosis@$IP" 'cat >> ~/.ssh/authorized_keys'
224 else
225 say "germinate: WARNING: ssh_pubkey is not one well-formed key — ignoring it"
226 fi
227 fi
228
229 say "germinate: streaming the node journal (==> markers) — budget 60m"
230 vm "sudo journalctl -u mitosis-node -f -n 200 2>/dev/null | grep --line-buffered -E '==>|FATAL'" &
231 TAIL_PID=$!
232 GERM_OK=""
233 for _ in $(seq 1 240); do
234 ST=$(vm "systemctl is-active mitosis-node" 2>/dev/null || true)
235 case "$ST" in
236 active) GERM_OK=1; break ;;
237 failed) kill "$TAIL_PID" 2>/dev/null || true; TAIL_PID=""
238 say "germinate: FAILED — last journal:"; vm "sudo journalctl -u mitosis-node -n 60 --no-pager" || true
239 died "germination failed on the VM" ;;
240 esac
241 sleep 15
242 done
243 kill "$TAIL_PID" 2>/dev/null || true; TAIL_PID=""
244 [ -n "$GERM_OK" ] || died "germination still running after 60m — treating as hang"
245 vm "sudo test -f /root/PLATFORM.md" || died "mitosis-node active but /root/PLATFORM.md missing"
246 say "germinate: complete — platform card written on the VM"
247
248 # assertions run ON the node (NODE_MODE=local: A5 probes the host directly)
249 run_gate() { # $1=label $2=timeout $3=assert-args
250 say "$1: running coldstart-assert on the VM (budget $(($2 / 60))m)"
251 # coldstart-assert opens a `kubectl port-forward` whose orphan can outlive
252 # the script and hold the SSH channel open forever (the phase would hang
253 # after a PASS). Reap it and return the assert's real exit code. Safe to
254 # pkill broadly here: the gate VM is single-tenant, one run only.
255 vm "sudo bash -c 'set +u; . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh; set -u; cd /opt/mitosis; export KUBECONFIG=/etc/rancher/k3s/k3s.yaml NODE_MODE=local; timeout $2 nix develop --command bash scripts/coldstart-assert.sh $3; rc=\$?; pkill -f \"port-forward svc/forgejo-http\" 2>/dev/null || true; exit \$rc'"
256 }
257 run_gate gate 1500 "default $DOMAIN" || died "coldstart gate FAILED on $DOMAIN"
258 say "gate: PASS"
259 if [ "$SKIP_E2E" = 1 ]; then say "e2e: skipped (--skip-e2e)"
260 elif [ -z "$CREW" ]; then say "e2e: SKIP — no crew credential sealed, cannot build"
261 else run_gate e2e 2400 "--e2e default $DOMAIN" || died "app E2E FAILED on $DOMAIN"; say "e2e: PASS — canary reached prod"; fi
262
263 # ═══════════════════════════════ AKS GROUND ═════════════════════════════════
264 else
265 [ -f "$REPO/genesis/seed.tar.gz" ] || died "aks mode needs a seed at genesis/seed.tar.gz"
266 command -v kubectl >/dev/null || died "aks mode needs kubectl on PATH"
267 cf_load
268 DOMAIN="$SLUG.$GATE_BASE_DOMAIN"
269
270 say "up: deploying main.bicep (mode=aks, domain=$DOMAIN) — a few minutes"
271 az deployment group create -g "$RG" -n gate -f "$HERE/main.bicep" --output none \
272 -p domain="$DOMAIN" -p mode=aks
273 ACR=$(az deployment group show -g "$RG" -n gate --query 'properties.outputs.acrLoginServer.value' -o tsv)
274 [ -n "$ACR" ] || died "aks deployment produced no ACR login server"
275 # delegate <run>.gate.open-platform.sh → the fresh Azure zone's nameservers
276 NS=()
277 while IFS= read -r _ns; do [ -n "$_ns" ] && NS+=("$_ns"); done \
278 < <(az network dns zone show -g "$RG" -n "$DOMAIN" --query nameServers -o tsv | tr -d '\r')
279 [ "${#NS[@]}" -ge 1 ] || died "Azure DNS zone $DOMAIN has no nameservers"
280 cf_delegate "$DOMAIN" "${NS[@]}"
281
282 say "up: fetching kubeconfig (az aks get-credentials)"
283 az aks get-credentials -g "$RG" -n mitosis-aks --file "$KUBECONFIG" --overwrite-existing --only-show-errors
284 CTX=mitosis-aks
285
286 say "up: mirroring infra images into ACR $ACR (az acr import, ~2-4m)"
287 bash "$HERE/mirror-images.sh" "$ACR" || died "image mirror into ACR failed"
288
289 # germinate LOCALLY against the cluster; watch for the LB IP to write edge records
290 say "germinate: starting (CLUSTER_PROFILE=aks) — budget 75m; edge A records land when the LB IP appears"
291 ( cd "$REPO"
292 export DOMAIN CLUSTER_PROFILE=aks ACR="$ACR" AKS_RG="$RG" FORK_KEY_ACK=1
293 export SEED="$REPO/genesis/seed.tar.gz"
294 export FORK_KEY="$WORK/fork-$DOMAIN.age"
295 export CLAUDE_CODE_OAUTH_TOKEN="$CREW"
296 # az (outer nix shell) + the flake devShell (nix develop) both on PATH for germinate
297 nix shell nixpkgs#azure-cli --command nix develop --command bash -c 'bin/germinate'
298 ) > "$WORK/germinate.log" 2>&1 &
299 GERM_PID=$!
300 tail -f "$WORK/germinate.log" 2>/dev/null | grep --line-buffered -E '==>|FATAL|ERROR' &
301 TAIL_PID=$!
302
303 # write the edge records as soon as traefik's LB has an external IP
304 EDGE_DONE=""
305 for _ in $(seq 1 300); do
306 kill -0 "$GERM_PID" 2>/dev/null || break # germinate exited (done or died) — stop waiting
307 if [ -z "$EDGE_DONE" ]; then
308 LBIP=$(kubectl --context "$CTX" -n traefik get svc traefik \
309 -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true)
310 if [ -n "$LBIP" ]; then
311 say "germinate: traefik LB is $LBIP — writing *.$DOMAIN and @ A records"
312 az network dns record-set a add-record -g "$RG" -z "$DOMAIN" -n '*' -a "$LBIP" --ttl 60 --output none || true
313 az network dns record-set a add-record -g "$RG" -z "$DOMAIN" -n '@' -a "$LBIP" --ttl 60 --output none || true
314 EDGE_DONE=1
315 fi
316 fi
317 sleep 15
318 done
319 wait "$GERM_PID"; GRC=$?; GERM_PID=""
320 kill "$TAIL_PID" 2>/dev/null || true; TAIL_PID=""
321 [ "$GRC" = 0 ] || { say "germinate: FAILED (rc=$GRC) — tail:"; tail -40 "$WORK/germinate.log"; died "aks germination failed"; }
322 [ -n "$EDGE_DONE" ] || say "germinate: NOTE: LB IP never observed — edge records may be missing"
323 say "germinate: complete"
324
325 run_gate() { # $1=label $2=timeout $3=assert-args (runs locally, GROUND=aks)
326 say "$1: running coldstart-assert (GROUND=aks) — budget $(($2 / 60))m"
327 ( cd "$REPO" && GROUND=aks _timeout "$2" nix develop --command bash -c "bash scripts/coldstart-assert.sh $3" )
328 }
329 run_gate gate 1500 "$CTX $DOMAIN" || died "coldstart gate FAILED on $DOMAIN"
330 say "gate: PASS"
331 if [ "$SKIP_E2E" = 1 ]; then say "e2e: skipped (--skip-e2e)"
332 elif [ -z "$CREW" ]; then say "e2e: SKIP — no crew credential sealed, cannot build"
333 else run_gate e2e 2400 "--e2e $CTX $DOMAIN" || died "app E2E FAILED on $DOMAIN"; say "e2e: PASS — canary reached prod"; fi
334 fi
335
336 # ── summary ──────────────────────────────────────────────────────────────────
337 E2E_NOTE=""; { [ "$SKIP_E2E" = 1 ] || [ -z "$CREW" ]; } || E2E_NOTE=", app E2E green"
338 say "RESULT: PASS ($MODE) — https://git.$DOMAIN germinated, asserted$E2E_NOTE"
339 [ "$KEEP" = 1 ] && say "kept: RG $RG · domain $DOMAIN · TTL $EXPIRES (card is on the platform, not printed — CI logs are public)"
docs/ops/coldstart-ci.md +20
@@ -200,3 +200,23 @@ matches the name part; `:host` selects host execution mode.
200200 current gitops, verifies it (`scripts/reseed-verify.sh`), and opens a reseed PR
201201 as the restricted `ci-bot` user; the PR's own coldstart gate then germinates
202202 from that exact seed before it can merge.
203
204 ## The Azure gate (real-cloud sibling)
205
206 The k3d gate proves germination in a container; `azure-gate` proves the whole
207 cloud path on real Azure — bicep deploy, node contract, zero-DNS sslip domain,
208 public edge, and the app E2E — then destroys everything. Same assertion script
209 (`coldstart-assert.sh` with `NODE_MODE=local`, run ON the VM over SSH).
210
211 | Piece | Where |
212 | -------- | ------------------------------------------------------------------ |
213 | Script | `deploy/azure/gate.sh` (laptop and CI run the SAME one) |
214 | Workflow | `.forgejo/workflows/azure-gate.yml` (dispatch + nightly janitor) |
215 | Identity | SP `mitosis-azure-gate`, Contributor on the subscription |
216 | Secrets | `AZURE_GATE_{CLIENT_ID,CLIENT_SECRET,TENANT_ID,SUBSCRIPTION_ID}`, `AZURE_GATE_CLAUDE_TOKEN` (crew credential for the E2E; absent = loud SKIP) |
217 | Cost | ~$0.30–0.60/run (RG-per-run, tagged; Cost Analysis groups by `run`) |
218
219 Dispatch-only (each run is real money); `keep=true` + your `ssh_pubkey` turns a
220 run into a live deployment you can use. Isolation: everything lives in
221 `mitosis-gate-<run>`; the janitor reaps any gate RG whose `expires` tag passed,
222 so crashed runs and forgotten keeps die within a day.
scripts/coldstart-assert.sh +47 −7
@@ -16,7 +16,8 @@
1616 #
1717 # Requires: bash, kubectl, curl, jq; docker for A5 (must reach the k3d node
1818 # container, i.e. run on the machine that hosts the cluster). bash 3.2
19 # compatible (macOS).
19 # compatible (macOS). On a bare-VM k3s ground (deploy/azure gate), run it ON
20 # the node with NODE_MODE=local — A5 probes the host directly, no docker.
2021 #
2122 # Assertions (each prints PASS/FAIL/SKIP + one evidence line):
2223 # A1 all Flux Kustomizations Ready=True (>=10, incl. kyverno + kyverno-policies);
@@ -87,6 +88,35 @@ WEBHOOK_LOG_MIN_VERSION="0.7.6" # agents version that logs "system webhook verif
8788
8889 k() { kubectl --context "$CTX" "$@"; }
8990
91 # Node-namespace probes (A5). Two grounds:
92 # k3d (default): the node is a docker container named ${CTX}-server-0 —
93 # docker exec reaches its netns and its crictl.
94 # bare VM (NODE_MODE=local): this script runs ON the k3s node itself
95 # (deploy/azure gate sshes in and runs it there) — no container exists;
96 # the host's own shell IS the node namespace, and crictl is k3s's.
97 NODE_MODE="${NODE_MODE:-k3d}"
98
99 # GROUND — which substrate the platform germinated on. k3s (default): the full
100 # recipe, every organ present. aks: a MANAGED Azure ground where
101 # trim_gitops_aks removed the k3s-only organs (monitoring/grafana, node-fabric,
102 # registry-node-config, coredns rewrites) — so A5 (the containerd-mirror probes)
103 # is skipped and the Flux-Kustomization floor is lower. Everything else asserts
104 # identically. A9-grafana already self-skips when the monitoring Kustomization
105 # is absent, so it needs no guard here.
106 GROUND="${GROUND:-k3s}"
107 KUST_MIN=10
108 [ "$GROUND" = aks ] && KUST_MIN=6 # aks drops monitoring/node-fabric/registry-node-config/coredns
109 node_sh() {
110 if [ "$NODE_MODE" = local ]; then sh -c "$1"; else docker exec "$NODE" sh -c "$1"; fi
111 }
112 node_crictl() {
113 if [ "$NODE_MODE" = local ]; then
114 if command -v crictl >/dev/null 2>&1; then crictl "$@"; else k3s crictl "$@"; fi
115 else
116 docker exec "$NODE" crictl "$@"
117 fi
118 }
119
90120 T0=$SECONDS
91121 FAIL_COUNT=0
92122 FAIL_LIST=""
@@ -385,7 +415,7 @@ while :; do
385415 KUST_NOT_READY=$(echo "$KUST" | awk 'NF && $2 != "True" {print $1"="$2}' | tr '\n' ' ')
386416 HAS_KYVERNO=$(echo "$KUST" | awk '$1 == "kyverno" {print 1}')
387417 HAS_KYVERNO_POLICIES=$(echo "$KUST" | awk '$1 == "kyverno-policies" {print 1}')
388 if [ "$KUST_TOTAL" -ge 10 ] && [ -z "$KUST_NOT_READY" ] &&
418 if [ "$KUST_TOTAL" -ge "$KUST_MIN" ] && [ -z "$KUST_NOT_READY" ] &&
389419 [ -n "$HAS_KYVERNO" ] && [ -n "$HAS_KYVERNO_POLICIES" ]; then
390420 break
391421 fi
@@ -395,11 +425,11 @@ while :; do
395425 sleep 10
396426 done
397427
398 if [ "$KUST_TOTAL" -ge 10 ] && [ -z "$KUST_NOT_READY" ] &&
428 if [ "$KUST_TOTAL" -ge "$KUST_MIN" ] && [ -z "$KUST_NOT_READY" ] &&
399429 [ -n "$HAS_KYVERNO" ] && [ -n "$HAS_KYVERNO_POLICIES" ]; then
400430 pass A1-kustomizations "${KUST_TOTAL} Kustomizations all Ready=True, incl. kyverno + kyverno-policies: $(echo "$KUST" | awk '{print $1}' | tr '\n' ' ')"
401431 else
402 fail A1-kustomizations "total=${KUST_TOTAL} (want >=10) notReady=[${KUST_NOT_READY:-none}] kyverno=${HAS_KYVERNO:-missing} kyverno-policies=${HAS_KYVERNO_POLICIES:-missing}"
432 fail A1-kustomizations "total=${KUST_TOTAL} (want >=${KUST_MIN}) notReady=[${KUST_NOT_READY:-none}] kyverno=${HAS_KYVERNO:-missing} kyverno-policies=${HAS_KYVERNO_POLICIES:-missing}"
403433 fi
404434
405435 POL_COUNT=$(k get clusterpolicies.kyverno.io --no-headers 2>/dev/null | grep -c . || true)
@@ -509,6 +539,14 @@ else
509539 fi
510540
511541 # ---------- A5: registry mirror --------------------------------------------------
542 # aks ground has no containerd mirror: trim_gitops_aks removes
543 # registry-node-config, nodes pull from ACR + git.<domain> over public TLS,
544 # and there is no reachable node namespace to probe. Skip all three sub-checks.
545 if [ "$GROUND" = aks ]; then
546 skip A5-forwarder-running "aks ground: registry-node-config removed by trim_gitops_aks"
547 skip A5-node-listener "aks ground: no containerd mirror / no reachable node namespace"
548 skip A5-mirror-pull "aks ground: images come from ACR + git.<domain> public TLS, not the :31100 mirror"
549 else
512550
513551 FWD_LINE=$(k get pods -A --no-headers 2>/dev/null | grep 'registry-node-config' | head -1 || true)
514552 if echo "$FWD_LINE" | grep -q 'Running'; then
@@ -518,7 +556,7 @@ else
518556 fi
519557
520558 NODE="${CTX}-server-0" # k3d context k3d-X -> node container k3d-X-server-0
521 LISTEN=$(docker exec "$NODE" sh -c 'ss -tln 2>/dev/null || netstat -tln' 2>/dev/null | grep '31100' | head -1 || true)
559 LISTEN=$(node_sh 'ss -tln 2>/dev/null || netstat -tln' 2>/dev/null | grep '31100' | head -1 || true)
522560 if [ -n "$LISTEN" ]; then
523561 pass A5-node-listener "node netns listener on :31100: ${LISTEN}"
524562 else
@@ -532,14 +570,16 @@ fi
532570 # fallback for old platforms where mcp may predate this convention.
533571 PULL_REF=$(k -n mcp get deploy mcp -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true)
534572 case "$PULL_REF" in "git.${DOMAIN}"/*) ;; *) PULL_REF="git.${DOMAIN}/plat/ci-builder:${CIB_TAG}" ;; esac
535 if docker exec "$NODE" crictl pull "$PULL_REF" >/dev/null 2>&1; then
573 if node_crictl pull "$PULL_REF" >/dev/null 2>&1; then
536574 pass A5-mirror-pull "crictl pull ${PULL_REF} succeeded (anonymous)"
537 elif [ -n "${ADMIN_PW:-}" ] && docker exec "$NODE" crictl pull --creds "${ADMIN_USER}:${ADMIN_PW}" "$PULL_REF" >/dev/null 2>&1; then
575 elif [ -n "${ADMIN_PW:-}" ] && node_crictl pull --creds "${ADMIN_USER}:${ADMIN_PW}" "$PULL_REF" >/dev/null 2>&1; then
538576 pass A5-mirror-pull "crictl pull ${PULL_REF} succeeded (with forgejo-admin creds)"
539577 else
540578 fail A5-mirror-pull "crictl pull ${PULL_REF} failed through the 127.0.0.1:31100 mirror"
541579 fi
542580
581 fi # end GROUND != aks (A5)
582
543583 # ---------- A6: dispatcher + mcp Available ---------------------------------------
544584
545585 AG_AVAIL=$(k -n agents get deploy agents -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null || true)

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs