Public: anyone can read this; nobody can change it.
workbenchthis machine
azure-gate: real-cloud CI gate on Azure (vms + aks), manual-dispatch #143
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
feat/azure-gate → main 9 commits
6 files
changed +720
−15
.forgejo/workflows/azure-gate.yml added +134
| @@ -0,0 +1,134 @@ | ||
| 1 | # azure-gate — deploy a REAL mitosis platform on Azure (deploy/azure/main.bicep, | |
| 2 | # mode=vms, domain=auto → zero DNS), run the cold-start gate + app E2E against | |
| 3 | # it over SSH, then destroy the resource group. ~$0.30-0.60 per run; per-run | |
| 4 | # spend is visible in Azure Cost Analysis grouped by tag `run`. | |
| 5 | # | |
| 6 | # Manual only (workflow_dispatch): each run costs real money and needs the | |
| 7 | # AZURE_GATE_* repo secrets, so no pull_request trigger. Inputs: | |
| 8 | # keep=true leave the platform running (a real deployment you can use); | |
| 9 | # pass ssh_pubkey so YOU can reach it — the gate's own key is | |
| 10 | # ephemeral. The janitor reaps it after the TTL (default 5h). | |
| 11 | # e2e=false skip the app-E2E phase (deploy + assert only) | |
| 12 | # ref defaults to the branch you dispatch from | |
| 13 | # | |
| 14 | # The nightly janitor job deletes any mitosis-gate-* RG whose tags.expires has | |
| 15 | # passed — leaked platforms (crashed runs, forgotten keeps) die within a day. | |
| 16 | # | |
| 17 | # All the logic lives in deploy/azure/gate.sh — run it from a laptop with an | |
| 18 | # az login and you get the identical cycle. Logs: every phase prints | |
| 19 | # `==> gate:` markers and the VM's germination journal (`==>` steps) streams | |
| 20 | # live into the job log. | |
| 21 | # | |
| 22 | # Secrets: AZURE_GATE_{CLIENT_ID,CLIENT_SECRET,TENANT_ID,SUBSCRIPTION_ID} | |
| 23 | # (service principal `mitosis-azure-gate`, Contributor on the sub), | |
| 24 | # AZURE_GATE_CLAUDE_TOKEN (crew credential sealed at germination — | |
| 25 | # absent = app E2E is a loud SKIP, deploy+assert still gate). | |
| 26 | ||
| 27 | name: azure-gate | |
| 28 | on: | |
| 29 | workflow_dispatch: | |
| 30 | inputs: | |
| 31 | mode: | |
| 32 | description: ground to test — vms (bare VM, ~$0.30) or aks (managed, ~$1-2) | |
| 33 | required: false | |
| 34 | default: vms | |
| 35 | type: choice | |
| 36 | options: [vms, aks] | |
| 37 | keep: | |
| 38 | description: keep the platform after the gate (true/false) | |
| 39 | required: false | |
| 40 | default: "false" | |
| 41 | e2e: | |
| 42 | description: run the app E2E phase (true/false) | |
| 43 | required: false | |
| 44 | default: "true" | |
| 45 | ssh_pubkey: | |
| 46 | description: your SSH public key, authorized on the VM (vms + keep=true) | |
| 47 | required: false | |
| 48 | default: "" | |
| 49 | schedule: | |
| 50 | - cron: "41 5 * * *" # nightly janitor: reap expired mitosis-gate-* RGs | |
| 51 | ||
| 52 | concurrency: | |
| 53 | group: azure-gate | |
| 54 | cancel-in-progress: false # a cancelled deploy would orphan a paid RG; runs queue instead | |
| 55 | ||
| 56 | jobs: | |
| 57 | gate: | |
| 58 | if: github.event_name == 'workflow_dispatch' | |
| 59 | runs-on: vxrail-coldstart | |
| 60 | timeout-minutes: 180 | |
| 61 | steps: | |
| 62 | - name: Clone (host runner has no node for actions/checkout) | |
| 63 | run: | | |
| 64 | set -euo pipefail | |
| 65 | rm -rf src | |
| 66 | git clone --depth 50 --branch "${{ github.ref_name }}" \ | |
| 67 | https://git.open-platform.sh/${{ github.repository }} src | |
| 68 | git -C src log -1 --oneline | |
| 69 | ||
| 70 | - name: Azure gate (deploy → assert → e2e → destroy) | |
| 71 | env: | |
| 72 | AZURE_GATE_CLIENT_ID: ${{ secrets.AZURE_GATE_CLIENT_ID }} | |
| 73 | AZURE_GATE_CLIENT_SECRET: ${{ secrets.AZURE_GATE_CLIENT_SECRET }} | |
| 74 | AZURE_GATE_TENANT_ID: ${{ secrets.AZURE_GATE_TENANT_ID }} | |
| 75 | AZURE_GATE_SUBSCRIPTION_ID: ${{ secrets.AZURE_GATE_SUBSCRIPTION_ID }} | |
| 76 | AZURE_GATE_CLAUDE_TOKEN: ${{ secrets.AZURE_GATE_CLAUDE_TOKEN }} | |
| 77 | CF_API_TOKEN: ${{ secrets.AZURE_GATE_CF_TOKEN }} | |
| 78 | GATE_RUN: r${{ github.run_number }}a${{ github.run_attempt }} | |
| 79 | MODE: ${{ github.event.inputs.mode }} | |
| 80 | MITOSIS_REF: ${{ github.ref_name }} | |
| 81 | GATE_SSH_PUBKEY: ${{ github.event.inputs.ssh_pubkey }} | |
| 82 | # az config lives under the job workspace, not the runner user's home | |
| 83 | AZURE_CONFIG_DIR: ${{ github.workspace }}/.azure | |
| 84 | run: | | |
| 85 | set -euo pipefail | |
| 86 | ARGS="" | |
| 87 | [ "${{ github.event.inputs.keep }}" = "true" ] && ARGS="$ARGS --keep" | |
| 88 | [ "${{ github.event.inputs.e2e }}" = "false" ] && ARGS="$ARGS --skip-e2e" | |
| 89 | # Orchestrator tools on PATH; gate.sh layers `nix develop` (flake tools) | |
| 90 | # for germinate + coldstart-assert. az/kubectl/jq/ssh/curl cover both modes. | |
| 91 | nix shell nixpkgs#azure-cli nixpkgs#kubectl nixpkgs#jq nixpkgs#openssh nixpkgs#curl --command \ | |
| 92 | bash src/deploy/azure/gate.sh $ARGS | |
| 93 | ||
| 94 | - name: Safety net — the RG must be gone (unless keep) | |
| 95 | if: always() | |
| 96 | env: | |
| 97 | AZURE_GATE_CLIENT_ID: ${{ secrets.AZURE_GATE_CLIENT_ID }} | |
| 98 | AZURE_GATE_CLIENT_SECRET: ${{ secrets.AZURE_GATE_CLIENT_SECRET }} | |
| 99 | AZURE_GATE_TENANT_ID: ${{ secrets.AZURE_GATE_TENANT_ID }} | |
| 100 | AZURE_GATE_SUBSCRIPTION_ID: ${{ secrets.AZURE_GATE_SUBSCRIPTION_ID }} | |
| 101 | AZURE_CONFIG_DIR: ${{ github.workspace }}/.azure | |
| 102 | run: | | |
| 103 | set -euo pipefail | |
| 104 | [ "${{ github.event.inputs.keep }}" = "true" ] && { echo "keep=true — RG intentionally alive (janitor TTL applies)"; exit 0; } | |
| 105 | nix shell nixpkgs#azure-cli --command bash -c ' | |
| 106 | az account show >/dev/null 2>&1 || az login --service-principal \ | |
| 107 | -u "$AZURE_GATE_CLIENT_ID" -p "$AZURE_GATE_CLIENT_SECRET" \ | |
| 108 | --tenant "$AZURE_GATE_TENANT_ID" --output none | |
| 109 | az account set -s "$AZURE_GATE_SUBSCRIPTION_ID" | |
| 110 | RG="mitosis-gate-r${{ github.run_number }}a${{ github.run_attempt }}" | |
| 111 | if az group exists -n "$RG" | grep -qi true; then | |
| 112 | echo "RG $RG still exists — deleting (safety net)" | |
| 113 | az group delete -n "$RG" --yes --no-wait | |
| 114 | else | |
| 115 | echo "RG $RG gone — clean" | |
| 116 | fi' | |
| 117 | ||
| 118 | janitor: | |
| 119 | if: github.event_name == 'schedule' | |
| 120 | runs-on: vxrail-coldstart | |
| 121 | timeout-minutes: 20 | |
| 122 | steps: | |
| 123 | - name: Reap expired mitosis-gate-* resource groups | |
| 124 | env: | |
| 125 | AZURE_GATE_CLIENT_ID: ${{ secrets.AZURE_GATE_CLIENT_ID }} | |
| 126 | AZURE_GATE_CLIENT_SECRET: ${{ secrets.AZURE_GATE_CLIENT_SECRET }} | |
| 127 | AZURE_GATE_TENANT_ID: ${{ secrets.AZURE_GATE_TENANT_ID }} | |
| 128 | AZURE_GATE_SUBSCRIPTION_ID: ${{ secrets.AZURE_GATE_SUBSCRIPTION_ID }} | |
| 129 | AZURE_CONFIG_DIR: ${{ github.workspace }}/.azure | |
| 130 | run: | | |
| 131 | set -euo pipefail | |
| 132 | rm -rf src | |
| 133 | git clone --depth 1 https://git.open-platform.sh/${{ github.repository }} src | |
| 134 | nix shell nixpkgs#azure-cli --command bash src/deploy/azure/gate.sh --janitor |
deploy/azure/GATE-NOTES.md added +120
| @@ -0,0 +1,120 @@ | ||
| 1 | # Azure gate — build notes & handoff | |
| 2 | ||
| 3 | Running log for the `azure-gate` CI (deploy a real mitosis platform on Azure, | |
| 4 | test it, destroy it). Written so Ben (and his agent) can pick this up. Status | |
| 5 | markers: ✅ done · 🚧 in progress · ⬜ todo. | |
| 6 | ||
| 7 | ## What it is | |
| 8 | ||
| 9 | One workflow `.forgejo/workflows/azure-gate.yml` + one script | |
| 10 | `deploy/azure/gate.sh` that deploys `deploy/azure/main.bicep`, waits for | |
| 11 | germination, runs `scripts/coldstart-assert.sh` (+ `--e2e`), then destroys the | |
| 12 | resource group. Two grounds, chosen per run: | |
| 13 | ||
| 14 | - **`vms`** (✅ working) — `mode=vms`, `domain=auto` (sslip.io, zero DNS). | |
| 15 | ~$0.30/run, ~1h. The autonomous smoke test. | |
| 16 | - **`aks`** (🚧) — `mode=aks`, real sub-domain of `open-platform.sh` delegated | |
| 17 | to a per-run Azure DNS zone. ~$1–2/run, ~2h. Tests the product ground. | |
| 18 | ||
| 19 | `gate.sh` runs identically from a laptop (`az login`) and from CI (service | |
| 20 | principal). All logic in the script; the workflow stays declarative. | |
| 21 | ||
| 22 | ## Identity & secrets (already provisioned) | |
| 23 | ||
| 24 | - **Service principal** `mitosis-azure-gate`, appId | |
| 25 | `e9097ccc-2eb7-4b27-a461-b50baa8ef322`, **Contributor** on subscription | |
| 26 | `1d655bbf-4fc9-48dd-bf75-2d3f72bb64a1`. Rotate with | |
| 27 | `az ad sp credential reset --id <appId>`. | |
| 28 | - **Repo Actions secrets** (plat/mitosis): `AZURE_GATE_CLIENT_ID`, | |
| 29 | `AZURE_GATE_CLIENT_SECRET`, `AZURE_GATE_TENANT_ID`, | |
| 30 | `AZURE_GATE_SUBSCRIPTION_ID`, `AZURE_GATE_CLAUDE_TOKEN` (crew credential for | |
| 31 | the app E2E — absent = E2E is a loud SKIP, deploy+assert still gate). | |
| 32 | Set them with the `plat` admin PAT (ci-bot lacks the secrets scope): | |
| 33 | `curl -u plat:<PAT> -X PUT .../repos/plat/mitosis/actions/secrets/<NAME> -d '{"data":"..."}'`. | |
| 34 | - **Cloudflare token** for `open-platform.sh` lives at repo-root `cf-token` | |
| 35 | (gitignored — NEVER commit). Zone id `2fbcb5331b09d502067702811494d9be`. | |
| 36 | Used only by the `aks` path for per-run sub-domain delegation. | |
| 37 | ||
| 38 | ## DNS design (aks) — proven | |
| 39 | ||
| 40 | Per run, fully self-cleaning, no standing zone: | |
| 41 | ||
| 42 | 1. `main.bicep mode=aks createDnsZone=true` creates an Azure DNS zone named | |
| 43 | `<run>.gate.open-platform.sh` and outputs its 4 `dnsNameServers`. | |
| 44 | 2. `gate.sh` adds those as **NS records** under `open-platform.sh` in | |
| 45 | Cloudflare (delegating `<run>.gate.open-platform.sh` → Azure). | |
| 46 | 3. cert-manager does DNS-01 in the Azure zone (public LE cert). The | |
| 47 | `letsencrypt.org` CAA on `open-platform.sh` is inherited → wildcard | |
| 48 | issuance allowed. ✅ verified. | |
| 49 | 4. When the traefik LB gets its external IP, `gate.sh` writes `*` and `@` A | |
| 50 | records into the Azure zone. | |
| 51 | 5. Teardown deletes the RG (takes the Azure zone with it) **and** removes the | |
| 52 | Cloudflare NS records. | |
| 53 | ||
| 54 | **Proof done 2026-08-19**: throwaway zone `dnstest.gate.open-platform.sh` | |
| 55 | delegated via CF, resolved through Azure on both 1.1.1.1 and 8.8.8.8, torn | |
| 56 | down clean. The `*.open-platform.sh` proxied tunnel CNAME does NOT shadow it | |
| 57 | (wildcards match one label only). | |
| 58 | ||
| 59 | ## Isolation & cleanup (both grounds) | |
| 60 | ||
| 61 | - Every run = its own RG `mitosis-gate-<run>` (CI: `r<runNumber>a<attempt>`), | |
| 62 | tagged `purpose=azure-gate run=<> ref=<> expires=<iso>`. | |
| 63 | - `gate.sh` teardown runs on **any** exit unless `--keep`. | |
| 64 | - Workflow has an `if: always()` safety-net step that force-deletes the RG. | |
| 65 | - Nightly **janitor** (`gate.sh --janitor`, cron in the workflow) deletes any | |
| 66 | `mitosis-gate-*` RG whose `expires` tag has passed — reaps crashed runs and | |
| 67 | forgotten `--keep`s within a day. | |
| 68 | - Cost: group Azure Cost Analysis by tag `run` for per-run spend. | |
| 69 | ||
| 70 | ## Security | |
| 71 | ||
| 72 | - `ssh_pubkey` workflow input is validated to one well-formed key line and | |
| 73 | piped over stdin (never interpolated into the remote command) — an | |
| 74 | adversarial review caught a command-injection → root-RCE here; fixed. | |
| 75 | - `claudeCodeOauthToken` is `@secure()` in bicep (not in ARM history). The | |
| 76 | gate VM logs are streamed to the (public) CI log filtered to `==>|FATAL`, so | |
| 77 | no card/secret content is printed. `--keep` does NOT print the card. | |
| 78 | ||
| 79 | ## Gotchas found while proving it (2026-08-19) | |
| 80 | ||
| 81 | - **SSH gate phase hangs after a PASS (vms).** `coldstart-assert` opens a | |
| 82 | `kubectl port-forward`; its process can orphan (reparent to init) and hold | |
| 83 | the SSH channel open, so `run_gate` never returns even though the assert | |
| 84 | exited 0. Fix: the remote command reaps `port-forward svc/forgejo-http` and | |
| 85 | returns the assert's real rc; SSH keepalive drops a dead channel. Do NOT | |
| 86 | move the `pkill` into `coldstart-assert.sh` — the k3d coldstart gate runs | |
| 87 | many in parallel on one host and a broad pkill would kill siblings. The | |
| 88 | gate VM is single-tenant, so pkill there is safe. (aks is unaffected — its | |
| 89 | assert runs locally, no SSH channel.) | |
| 90 | - `pgrep -f GATE_RUN=l3` finds nothing even when alive: `env VAR=x` puts the | |
| 91 | var in the environment, not argv. Match on `gate.sh --mode aks` instead. | |
| 92 | ||
| 93 | ## State (2026-08-19) | |
| 94 | ||
| 95 | - ✅ vms `gate.sh` + workflow, committed on `feat/azure-gate`. | |
| 96 | - ✅ `coldstart-assert.sh` `NODE_MODE=local` (bare-VM A5; k3d path unchanged). | |
| 97 | - ✅ Azure SP + 5 repo secrets + cf-token confirmed. | |
| 98 | - ✅ DNS per-run delegation proven end-to-end. | |
| 99 | - ✅ Local vms proof run (`l2`, leader 52.188.125.153): germination + the FULL | |
| 100 | cold-start gate PASSED on real Azure — A1–A12 green, including A5 via | |
| 101 | `NODE_MODE=local`. app E2E running at time of writing. | |
| 102 | - ✅ aks path in `gate.sh` (`--mode aks`) + `GROUND=aks` guards in | |
| 103 | `coldstart-assert.sh` (skip A5; lower Kustomization floor `KUST_MIN`; | |
| 104 | A9-grafana self-skips). shellcheck-clean, k3s path byte-identical. | |
| 105 | - 🚧 Local aks proof run (`l3`) in flight — first end-to-end aks exercise; | |
| 106 | iterating on real failures. | |
| 107 | - ⬜ Known risk: `aks-image-redirect` mutates the Pod but not the CNPG | |
| 108 | `Cluster` spec → tenant DBs restart-loop (from `aks-cost-experiment.md`). | |
| 109 | The app E2E on aks will likely surface this; fix is to extend the Kyverno | |
| 110 | policy to `clusters.postgresql.cnpg.io` spec.imageName. | |
| 111 | - ⬜ Open decision (from README truth pass): `mode=aks` bicep defaults still | |
| 112 | provision the $364/mo baseline (2×D4ds_v6 + Container Insights + managed | |
| 113 | Prometheus). Fine for a short-lived gate; revisit for the shipped default. | |
| 114 | ||
| 115 | ## How to run | |
| 116 | ||
| 117 | - CI: Forgejo → Actions → `azure-gate` → Run workflow. Inputs: `mode` (vms|aks), | |
| 118 | `keep`, `e2e`, `ssh_pubkey`. | |
| 119 | - Laptop: `deploy/azure/gate.sh [--mode aks] [--keep] [--skip-e2e]` with an | |
| 120 | `az login`. For aks it also reads `cf-token` at repo root. |
deploy/azure/README.md +60 −8
| @@ -396,13 +396,65 @@ NOTE (`aks` mode): the Key Vault has soft delete on. Purge it before you | ||
| 396 | 396 | create a new vault with the same name: |
| 397 | 397 | `az keyvault purge -n <keyVaultName>`. |
| 398 | 398 | |
| 399 | ## The Azure CI gate | |
| 400 | ||
| 401 | `azure-gate` deploys a real mitosis platform on Azure, tests it end to end | |
| 402 | (germinate → cold-start assertions → build a canary app and follow it to | |
| 403 | prod), then destroys everything. Each run is its own resource group, so runs | |
| 404 | never collide and cleanup is one delete. | |
| 405 | ||
| 406 | ### Run it (Forgejo) | |
| 407 | ||
| 408 | **Actions → azure-gate → Run workflow.** Choose: | |
| 409 | ||
| 410 | | Input | Meaning | | |
| 411 | | ------------ | ------------------------------------------------------------------------ | | |
| 412 | | `mode` | `vms` (bare VM, ~$0.30, ~1h) or `aks` (managed cluster, ~$1–2, ~2h) | | |
| 413 | | `e2e` | `true` also builds + tests a canary app; `false` = deploy + assert only | | |
| 414 | | `keep` | `true` leaves the platform running (a real deployment you can use) | | |
| 415 | | `ssh_pubkey` | for `vms` + `keep`: paste your `~/.ssh/id_ed25519.pub` so you can log in | | |
| 416 | ||
| 417 | Watch the live log — each phase prints `==> gate:` and the germination steps | |
| 418 | stream in. A green run means deployed, tested, and torn down. A `keep` run | |
| 419 | ends by printing how to reach the platform. | |
| 420 | ||
| 421 | ### Run it (your laptop) | |
| 422 | ||
| 423 | `az login`, then from the repo root: | |
| 424 | ||
| 425 | sh | |
| 426 | deploy/azure/gate.sh # vms, full cycle | |
| 427 | deploy/azure/gate.sh --mode aks # aks, full cycle | |
| 428 | deploy/azure/gate.sh --keep # keep the platform running | |
| 429 | deploy/azure/gate.sh --mode aks --skip-e2e # deploy + assert only | |
| 430 | ||
| 431 | ||
| 432 | `aks` mode also needs the `cf-token` file at the repo root (delegates a | |
| 433 | per-run sub-domain of `gate.open-platform.sh`). | |
| 434 | ||
| 435 | ### Reach or remove a kept platform | |
| 436 | ||
| 437 | - `vms`: `ssh mitosis@<ip>` (the run prints the IP; logins are in | |
| 438 | `sudo cat /root/platform-card.txt` on the VM). | |
| 439 | - `aks`: `az aks get-credentials -g mitosis-gate-<run> -n mitosis-aks`, then | |
| 440 | `kubectl`. | |
| 441 | - Remove it: `az group delete -n mitosis-gate-<run> --yes`. If you forget, | |
| 442 | the nightly janitor deletes any gate resource group whose `expires` tag has | |
| 443 | passed (default 5h). | |
| 444 | ||
| 445 | ### Cost | |
| 446 | ||
| 447 | In Azure **Cost analysis**, group by tag `run` to see each run's spend. A | |
| 448 | `vms` run is a few tens of cents; an `aks` run is a dollar or two. | |
| 449 | ||
| 399 | 450 | ## Files in this directory |
| 400 | 451 | |
| 401 | | File | Purpose | | |
| 402 | | ------------------------- | ----------------------------------------------------------- | | |
| 403 | | `main.bicep` | The deployment template. Both modes. | | |
| 404 | | `main.bicepparam.example` | Example parameter file. Copy, edit, do not commit the copy. | | |
| 405 | | `servers.bicep` | Add two k3s servers to a live `vms` platform. | | |
| 406 | | `mirror-images.sh` | Copy the infra images into the ACR (`aks` mode). | | |
| 407 | | `entra-forgejo-oidc.sh` | Add Entra ID sign-in to Forgejo (`aks` mode). | | |
| 408 | | `aks-cost-experiment.md` | Record of the first AKS platform: costs, commands, lessons. | | |
| 452 | | File | Purpose | | |
| 453 | | ------------------------- | ---------------------------------------------------------------------------- | | |
| 454 | | `main.bicep` | The deployment template. Both modes. | | |
| 455 | | `main.bicepparam.example` | Example parameter file. Copy, edit, do not commit the copy. | | |
| 456 | | `servers.bicep` | Add two k3s servers to a live `vms` platform. | | |
| 457 | | `mirror-images.sh` | Copy the infra images into the ACR (`aks` mode). | | |
| 458 | | `entra-forgejo-oidc.sh` | Add Entra ID sign-in to Forgejo (`aks` mode). | | |
| 459 | | `aks-cost-experiment.md` | Record of the first AKS platform: costs, commands, lessons. | | |
| 460 | | `gate.sh` | The Azure CI gate: deploy, test, destroy. Used by the `azure-gate` workflow. | |
deploy/azure/gate.sh added +339
| @@ -0,0 +1,339 @@ | ||
| 1 | #!/usr/bin/env bash | |
| 2 | # gate.sh — the Azure cloud gate: deploy a REAL mitosis platform on Azure, | |
| 3 | # assert it cold-start-healthy, run the app E2E (crew builds a canary app → | |
| 4 | # preview → merge → prod), then destroy the resource group. One script, two | |
| 5 | # grounds, runs identically from a laptop and from CI | |
| 6 | # (.forgejo/workflows/azure-gate.yml). | |
| 7 | # | |
| 8 | # deploy/azure/gate.sh # vms ground, full cycle | |
| 9 | # deploy/azure/gate.sh --mode aks # aks (managed) ground, full cycle | |
| 10 | # deploy/azure/gate.sh --keep # leave the platform running | |
| 11 | # deploy/azure/gate.sh --skip-e2e # deploy + assert only | |
| 12 | # deploy/azure/gate.sh --janitor # reap expired gate RGs, then exit | |
| 13 | # | |
| 14 | # Grounds: | |
| 15 | # vms (default) — main.bicep mode=vms, domain=auto (sslip.io → zero DNS). The | |
| 16 | # leader VM germinates; the gate watches + asserts over SSH. ~$0.30, ~1h. | |
| 17 | # aks — main.bicep mode=aks + a per-run Azure DNS zone | |
| 18 | # <run>.gate.open-platform.sh, delegated in Cloudflare (cf-token at repo | |
| 19 | # root). The gate germinates locally against the cluster kubeconfig | |
| 20 | # (CLUSTER_PROFILE=aks), writes the edge A records once the LB IP appears, | |
| 21 | # asserts with GROUND=aks. ~$1-2, ~2h. | |
| 22 | # | |
| 23 | # Auth: current `az` login if present; else the service principal in | |
| 24 | # AZURE_GATE_CLIENT_ID/_CLIENT_SECRET/_TENANT_ID (+ _SUBSCRIPTION_ID). CI | |
| 25 | # provides these from repo secrets. | |
| 26 | # | |
| 27 | # Env knobs: | |
| 28 | # MODE / --mode vms | aks (default vms) | |
| 29 | # GATE_RUN run id → RG mitosis-gate-$GATE_RUN (default: stamp+pid) | |
| 30 | # MITOSIS_REF branch/tag the platform is built from (default: main) | |
| 31 | # LOCATION Azure region (default: eastus) | |
| 32 | # VM_SIZE vms leader size (default Standard_D4as_v7) | |
| 33 | # GATE_BASE_DOMAIN aks parent domain (default gate.open-platform.sh) | |
| 34 | # CF_TOKEN_FILE Cloudflare token path (default: <repo>/cf-token) | |
| 35 | # AZURE_GATE_CLAUDE_TOKEN crew credential sealed at germination; absent = E2E SKIP | |
| 36 | # GATE_TTL_HOURS expiry tag for the janitor (default 5) | |
| 37 | # | |
| 38 | # Every phase prints `==> gate:` markers; the platform's own `==>` germination | |
| 39 | # log streams live into this log. Resources carry tags (purpose/run/ref/expires) | |
| 40 | # so Cost Analysis groups per-run spend and the janitor reaps leaks. | |
| 41 | set -euo pipefail | |
| 42 | ||
| 43 | HERE= | |
| 44 | REPO= | |
| 45 | KEEP=0 SKIP_E2E=0 JANITOR=0 | |
| 46 | MODE="" | |
| 47 | while [ -gt 0 ]; do case "$1" in | |
| 48 | --mode) MODE="$2"; shift 2 ;; | |
| 49 | --mode=*) MODE=""; shift ;; | |
| 50 | --vms) MODE=vms; shift ;; | |
| 51 | --aks) MODE=aks; shift ;; | |
| 52 | --keep) KEEP=1; shift ;; | |
| 53 | --skip-e2e) SKIP_E2E=1; shift ;; | |
| 54 | --janitor) JANITOR=1; shift ;; | |
| 55 | *) echo "unknown arg: $1" >&2; exit 2 ;; | |
| 56 | esac; done | |
| 57 | case "" in vms|aks) ;; *) echo "MODE must be vms or aks (got: )" >&2; exit 2 ;; esac | |
| 58 | ||
| 59 | : "" | |
| 60 | : "" | |
| 61 | : "" | |
| 62 | : "" | |
| 63 | : "" | |
| 64 | : "" | |
| 65 | GATE_RUN="" | |
| 66 | # DNS-label-safe run slug (aks subdomain): lowercase alnum, must start alpha. | |
| 67 | SLUG= | |
| 68 | RG="mitosis-gate-" | |
| 69 | T0= | |
| 70 | say() { echo "==> gate: $*"; } | |
| 71 | died() { echo "==> gate: FATAL: $*" >&2; exit 1; } | |
| 72 | # portable timeout (macOS has neither `timeout` nor, by default, `gtimeout`) | |
| 73 | if command -v timeout >/dev/null 2>&1; then _timeout() { timeout "$@"; } | |
| 74 | elif command -v gtimeout >/dev/null 2>&1; then _timeout() { gtimeout "$@"; } | |
| 75 | else _timeout() { shift; "$@"; }; fi # no timeout available: run without a cap | |
| 76 | ||
| 77 | # ── auth ───────────────────────────────────────────────────────────────────── | |
| 78 | command -v az >/dev/null || died "az CLI not found on PATH" | |
| 79 | if ! az account show >/dev/null 2>&1; then | |
| 80 | : "" | |
| 81 | say "logging in as service principal " | |
| 82 | az login --service-principal -u "" \ | |
| 83 | -p "" --tenant "" --output none | |
| 84 | fi | |
| 85 | [ -n "" ] && az account set -s "" | |
| 86 | SUB= | |
| 87 | say "mode= subscription= region=" | |
| 88 | ||
| 89 | # ── Cloudflare helpers (aks per-run sub-domain delegation) ─────────────────── | |
| 90 | CF_ZONE_ID="" CF_TOKEN="" | |
| 91 | cf_load() { | |
| 92 | # CI passes the token as CF_API_TOKEN (repo secret); a laptop uses the | |
| 93 | # gitignored repo-root cf-token file. | |
| 94 | if [ -n "" ]; then CF_TOKEN="" | |
| 95 | elif [ -f "" ]; then CF_TOKEN= | |
| 96 | else died "aks mode needs CF_API_TOKEN or a token file at (delegates )"; fi | |
| 97 | local apex="" # gate.open-platform.sh -> open-platform.sh | |
| 98 | CF_ZONE_ID= | |
| 99 | "https://api.cloudflare.com/client/v4/zones?name=" \ | |
| 100 | | python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r[0]["id"] if r else "")') | |
| 101 | [ -n "" ] || died "Cloudflare zone for not found with this token" | |
| 102 | } | |
| 103 | cf() { curl -fsS -H "Authorization: Bearer " -H "Content-Type: application/json" "$@"; } | |
| 104 | cf_delegate() { # $1=fqdn, remaining args = azure NS hosts | |
| 105 | local fqdn="$1"; shift | |
| 106 | local ns | |
| 107 | for ns in "$@"; do | |
| 108 | ns= | |
| 109 | cf -X POST "https://api.cloudflare.com/client/v4/zones//dns_records" \ | |
| 110 | -d "{\"type\":\"NS\",\"name\":\"\",\"content\":\"\",\"ttl\":120}" >/dev/null \ | |
| 111 | && say "dns: delegated -> " || say "dns: WARNING: NS create failed for " | |
| 112 | done | |
| 113 | } | |
| 114 | cf_undelegate() { # $1=fqdn — remove every NS record we added | |
| 115 | local fqdn="$1" id | |
| 116 | for id in | |
| 117 | | python3 -c 'import sys,json;[print(r["id"]) for r in json.load(sys.stdin).get("result",[])]' 2>/dev/null); do | |
| 118 | cf -X DELETE "https://api.cloudflare.com/client/v4/zones//dns_records/" >/dev/null \ | |
| 119 | && say "dns: removed delegation record " || true | |
| 120 | done | |
| 121 | } | |
| 122 | ||
| 123 | # ── janitor mode: reap expired gate RGs, then exit ─────────────────────────── | |
| 124 | if [ "" = 1 ]; then | |
| 125 | NOW= | |
| 126 | say "janitor: reaping RGs tagged purpose=azure-gate with expires < " | |
| 127 | REAPED=0 | |
| 128 | while IFS=$'\t' read -r name expires basedom; do | |
| 129 | [ -n "" ] || continue | |
| 130 | case "" in mitosis-gate-*) ;; *) say "janitor: not mitosis-gate-* — leaving it"; continue ;; esac | |
| 131 | if [[ "" < "" ]]; then | |
| 132 | say "janitor: deleting expired (expired )" | |
| 133 | # remove any Cloudflare delegation this run left behind | |
| 134 | if [ -n "" ] && [ "" != None ]; then | |
| 135 | cf_load 2>/dev/null && cf_undelegate "" || true | |
| 136 | fi | |
| 137 | az group delete -n "" --yes --no-wait | |
| 138 | REAPED= | |
| 139 | else | |
| 140 | say "janitor: alive until — leaving it" | |
| 141 | fi | |
| 142 | done < <(az group list --tag purpose=azure-gate --query "[].[name, tags.expires, tags.basedomain]" -o tsv) | |
| 143 | say "janitor: done ( deletion(s) started)" | |
| 144 | exit 0 | |
| 145 | fi | |
| 146 | ||
| 147 | # ── work dir + ephemeral ssh key (vms) ─────────────────────────────────────── | |
| 148 | WORK= | |
| 149 | trap 'rm -rf "$WORK"' EXIT # replaced by the full on_exit trap once teardown is defined | |
| 150 | export KUBECONFIG="/kubeconfig" | |
| 151 | SSH_KEY="/gate-key" | |
| 152 | ssh-keygen -q -t ed25519 -f "" -N "" -C "azure-gate-" | |
| 153 | SSH_OPTS=(-i "" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null | |
| 154 | -o ConnectTimeout=10 -o BatchMode=yes -o LogLevel=ERROR | |
| 155 | -o ServerAliveInterval=20 -o ServerAliveCountMax=6) # drop a truly dead channel | |
| 156 | vm() { ssh "" "mitosis@" "$@"; } | |
| 157 | ||
| 158 | # ── teardown: runs on ANY exit unless --keep ───────────────────────────────── | |
| 159 | DOWN_DONE=0 | |
| 160 | down() { | |
| 161 | [ "" = 1 ] && return 0 | |
| 162 | DOWN_DONE=1 | |
| 163 | [ "" = aks ] && [ -n "" ] && cf_undelegate "" || true | |
| 164 | say "down: deleting resource group " | |
| 165 | az group delete -n "" --yes --no-wait 2>/dev/null || true | |
| 166 | for _ in ; do | |
| 167 | az group exists -n "" 2>/dev/null | grep -qi false && { say "down: gone"; return 0; } | |
| 168 | sleep 15 | |
| 169 | done | |
| 170 | say "down: WARNING: still deleting after 15m — the janitor will confirm" | |
| 171 | } | |
| 172 | on_exit() { | |
| 173 | local rc= | |
| 174 | [ -n "" ] && kill "" 2>/dev/null || true | |
| 175 | [ -n "" ] && kill "" 2>/dev/null || true | |
| 176 | if [ "" = 1 ]; then | |
| 177 | say "keep: platform stays up. Reach it / destroy it:" | |
| 178 | [ "" = vms ] && echo " ssh mitosis@ # with the key you passed as ssh_pubkey" | |
| 179 | [ "" = aks ] && echo " az aks get-credentials -g -n mitosis-aks # then kubectl" | |
| 180 | echo " az group delete -n --yes # janitor reaps it after expires= regardless" | |
| 181 | else | |
| 182 | down | |
| 183 | fi | |
| 184 | rm -rf "" | |
| 185 | MIN= | |
| 186 | say "total wall clock m" | |
| 187 | exit | |
| 188 | } | |
| 189 | trap on_exit EXIT | |
| 190 | ||
| 191 | # ── RG create (shared) ─────────────────────────────────────────────────────── | |
| 192 | EXPIRES= | |
| 193 | || date -u -v+""H +%Y-%m-%dT%H:%M:%SZ) | |
| 194 | CREW="" | |
| 195 | say "up: creating (mode= ref= expires=)" | |
| 196 | BASETAG=""; [ "" = aks ] && BASETAG="." | |
| 197 | az group create -n "" -l "" --output none \ | |
| 198 | --tags purpose=azure-gate run="" ref="" expires="" \ | |
| 199 | mode="" basedomain="" | |
| 200 | ||
| 201 | # ═══════════════════════════════ VMS GROUND ═════════════════════════════════ | |
| 202 | if [ "" = vms ]; then | |
| 203 | MYIP= | |
| 204 | SSH_CIDR=""; SSH_CIDR="" | |
| 205 | [ "" = "*" ] && say "up: WARNING: could not learn our egress IP — SSH open to * for this run" | |
| 206 | ||
| 207 | say "up: deploying main.bicep (mode=vms, domain=auto, , ssh from )" | |
| 208 | az deployment group create -g "" -n gate -f "/main.bicep" --output none \ | |
| 209 | -p domain=auto -p adminSshPublicKey="$(cat ".pub")" -p vmSize="" \ | |
| 210 | -p allowedSshCidr="" -p mitosisRef="" -p claudeCodeOauthToken="" | |
| 211 | IP= | |
| 212 | [ -n "" ] || died "deployment produced no node IP" | |
| 213 | DOMAIN="$(echo "" | tr . -).sslip.io" | |
| 214 | say "up: leader → https://git." | |
| 215 | ||
| 216 | say "germinate: waiting for SSH on " | |
| 217 | for _ in ; do if vm true 2>/dev/null; then break; fi; sleep 15; done | |
| 218 | vm true || died "SSH to never came up" | |
| 219 | if [ -n "" ]; then | |
| 220 | if printf '%s' "" | grep -Eq '^(ssh-ed25519|ssh-rsa|ecdsa-sha2-[a-z0-9-]+|[email protected]|sk-ecdsa-sha2-[a-z0-9-][email protected]) [A-Za-z0-9+/=]+( [^\r\n]*)?$' \ | |
| 221 | && [ "$(printf '%s' "" | wc -l)" -eq 0 ]; then | |
| 222 | say "germinate: authorizing the operator's SSH key on the VM (for --keep)" | |
| 223 | printf '%s\n' "" | ssh "" "mitosis@" 'cat >> ~/.ssh/authorized_keys' | |
| 224 | else | |
| 225 | say "germinate: WARNING: ssh_pubkey is not one well-formed key — ignoring it" | |
| 226 | fi | |
| 227 | fi | |
| 228 | ||
| 229 | say "germinate: streaming the node journal (==> markers) — budget 60m" | |
| 230 | vm "sudo journalctl -u mitosis-node -f -n 200 2>/dev/null | grep --line-buffered -E '==>|FATAL'" & | |
| 231 | TAIL_PID=$! | |
| 232 | GERM_OK="" | |
| 233 | for _ in ; do | |
| 234 | ST= | |
| 235 | case "" in | |
| 236 | active) GERM_OK=1; break ;; | |
| 237 | failed) kill "" 2>/dev/null || true; TAIL_PID="" | |
| 238 | say "germinate: FAILED — last journal:"; vm "sudo journalctl -u mitosis-node -n 60 --no-pager" || true | |
| 239 | died "germination failed on the VM" ;; | |
| 240 | esac | |
| 241 | sleep 15 | |
| 242 | done | |
| 243 | kill "" 2>/dev/null || true; TAIL_PID="" | |
| 244 | [ -n "" ] || died "germination still running after 60m — treating as hang" | |
| 245 | vm "sudo test -f /root/PLATFORM.md" || died "mitosis-node active but /root/PLATFORM.md missing" | |
| 246 | say "germinate: complete — platform card written on the VM" | |
| 247 | ||
| 248 | # assertions run ON the node (NODE_MODE=local: A5 probes the host directly) | |
| 249 | run_gate() { # $1=label $2=timeout $3=assert-args | |
| 250 | say "$1: running coldstart-assert on the VM (budget $(($2 / 60))m)" | |
| 251 | # coldstart-assert opens a `kubectl port-forward` whose orphan can outlive | |
| 252 | # the script and hold the SSH channel open forever (the phase would hang | |
| 253 | # after a PASS). Reap it and return the assert's real exit code. Safe to | |
| 254 | # pkill broadly here: the gate VM is single-tenant, one run only. | |
| 255 | vm "sudo bash -c 'set +u; . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh; set -u; cd /opt/mitosis; export KUBECONFIG=/etc/rancher/k3s/k3s.yaml NODE_MODE=local; timeout $2 nix develop --command bash scripts/coldstart-assert.sh $3; rc=\$?; pkill -f \"port-forward svc/forgejo-http\" 2>/dev/null || true; exit \$rc'" | |
| 256 | } | |
| 257 | run_gate gate 1500 "default " || died "coldstart gate FAILED on " | |
| 258 | say "gate: PASS" | |
| 259 | if [ "" = 1 ]; then say "e2e: skipped (--skip-e2e)" | |
| 260 | elif [ -z "" ]; then say "e2e: SKIP — no crew credential sealed, cannot build" | |
| 261 | else run_gate e2e 2400 "--e2e default " || died "app E2E FAILED on "; say "e2e: PASS — canary reached prod"; fi | |
| 262 | ||
| 263 | # ═══════════════════════════════ AKS GROUND ═════════════════════════════════ | |
| 264 | else | |
| 265 | [ -f "/genesis/seed.tar.gz" ] || died "aks mode needs a seed at genesis/seed.tar.gz" | |
| 266 | command -v kubectl >/dev/null || died "aks mode needs kubectl on PATH" | |
| 267 | cf_load | |
| 268 | DOMAIN="." | |
| 269 | ||
| 270 | say "up: deploying main.bicep (mode=aks, domain=) — a few minutes" | |
| 271 | az deployment group create -g "" -n gate -f "/main.bicep" --output none \ | |
| 272 | -p domain="" -p mode=aks | |
| 273 | ACR= | |
| 274 | [ -n "" ] || died "aks deployment produced no ACR login server" | |
| 275 | # delegate <run>.gate.open-platform.sh → the fresh Azure zone's nameservers | |
| 276 | NS=() | |
| 277 | while IFS= read -r _ns; do [ -n "" ] && NS+=(""); done \ | |
| 278 | < <(az network dns zone show -g "" -n "" --query nameServers -o tsv | tr -d '\r') | |
| 279 | [ "" -ge 1 ] || died "Azure DNS zone has no nameservers" | |
| 280 | cf_delegate "" "" | |
| 281 | ||
| 282 | say "up: fetching kubeconfig (az aks get-credentials)" | |
| 283 | az aks get-credentials -g "" -n mitosis-aks --file "" --overwrite-existing --only-show-errors | |
| 284 | CTX=mitosis-aks | |
| 285 | ||
| 286 | say "up: mirroring infra images into ACR (az acr import, ~2-4m)" | |
| 287 | bash "/mirror-images.sh" "" || died "image mirror into ACR failed" | |
| 288 | ||
| 289 | # germinate LOCALLY against the cluster; watch for the LB IP to write edge records | |
| 290 | say "germinate: starting (CLUSTER_PROFILE=aks) — budget 75m; edge A records land when the LB IP appears" | |
| 291 | ( cd "" | |
| 292 | export DOMAIN CLUSTER_PROFILE=aks ACR="" AKS_RG="" FORK_KEY_ACK=1 | |
| 293 | export SEED="/genesis/seed.tar.gz" | |
| 294 | export FORK_KEY="/fork-.age" | |
| 295 | export CLAUDE_CODE_OAUTH_TOKEN="" | |
| 296 | # az (outer nix shell) + the flake devShell (nix develop) both on PATH for germinate | |
| 297 | nix shell nixpkgs#azure-cli --command nix develop --command bash -c 'bin/germinate' | |
| 298 | ) > "/germinate.log" 2>&1 & | |
| 299 | GERM_PID=$! | |
| 300 | tail -f "/germinate.log" 2>/dev/null | grep --line-buffered -E '==>|FATAL|ERROR' & | |
| 301 | TAIL_PID=$! | |
| 302 | ||
| 303 | # write the edge records as soon as traefik's LB has an external IP | |
| 304 | EDGE_DONE="" | |
| 305 | for _ in ; do | |
| 306 | kill -0 "" 2>/dev/null || break # germinate exited (done or died) — stop waiting | |
| 307 | if [ -z "" ]; then | |
| 308 | LBIP= | |
| 309 | -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true) | |
| 310 | if [ -n "" ]; then | |
| 311 | say "germinate: traefik LB is — writing *. and @ A records" | |
| 312 | az network dns record-set a add-record -g "" -z "" -n '*' -a "" --ttl 60 --output none || true | |
| 313 | az network dns record-set a add-record -g "" -z "" -n '@' -a "" --ttl 60 --output none || true | |
| 314 | EDGE_DONE=1 | |
| 315 | fi | |
| 316 | fi | |
| 317 | sleep 15 | |
| 318 | done | |
| 319 | wait ""; GRC=; GERM_PID="" | |
| 320 | kill "" 2>/dev/null || true; TAIL_PID="" | |
| 321 | [ "" = 0 ] || { say "germinate: FAILED (rc=) — tail:"; tail -40 "/germinate.log"; died "aks germination failed"; } | |
| 322 | [ -n "" ] || say "germinate: NOTE: LB IP never observed — edge records may be missing" | |
| 323 | say "germinate: complete" | |
| 324 | ||
| 325 | run_gate() { # $1=label $2=timeout $3=assert-args (runs locally, GROUND=aks) | |
| 326 | say "$1: running coldstart-assert (GROUND=aks) — budget $(($2 / 60))m" | |
| 327 | ( cd "" && GROUND=aks _timeout "$2" nix develop --command bash -c "bash scripts/coldstart-assert.sh $3" ) | |
| 328 | } | |
| 329 | run_gate gate 1500 " " || died "coldstart gate FAILED on " | |
| 330 | say "gate: PASS" | |
| 331 | if [ "" = 1 ]; then say "e2e: skipped (--skip-e2e)" | |
| 332 | elif [ -z "" ]; then say "e2e: SKIP — no crew credential sealed, cannot build" | |
| 333 | else run_gate e2e 2400 "--e2e " || died "app E2E FAILED on "; say "e2e: PASS — canary reached prod"; fi | |
| 334 | fi | |
| 335 | ||
| 336 | # ── summary ────────────────────────────────────────────────────────────────── | |
| 337 | E2E_NOTE=""; { [ "" = 1 ] || [ -z "" ]; } || E2E_NOTE=", app E2E green" | |
| 338 | say "RESULT: PASS () — https://git. germinated, asserted" | |
| 339 | [ "" = 1 ] && say "kept: RG · domain · TTL (card is on the platform, not printed — CI logs are public)" |
docs/ops/coldstart-ci.md +20
| @@ -200,3 +200,23 @@ matches the name part; `:host` selects host execution mode. | ||
| 200 | 200 | current gitops, verifies it (`scripts/reseed-verify.sh`), and opens a reseed PR |
| 201 | 201 | as the restricted `ci-bot` user; the PR's own coldstart gate then germinates |
| 202 | 202 | from that exact seed before it can merge. |
| 203 | ||
| 204 | ## The Azure gate (real-cloud sibling) | |
| 205 | ||
| 206 | The k3d gate proves germination in a container; `azure-gate` proves the whole | |
| 207 | cloud path on real Azure — bicep deploy, node contract, zero-DNS sslip domain, | |
| 208 | public edge, and the app E2E — then destroys everything. Same assertion script | |
| 209 | (`coldstart-assert.sh` with `NODE_MODE=local`, run ON the VM over SSH). | |
| 210 | ||
| 211 | | Piece | Where | | |
| 212 | | -------- | ------------------------------------------------------------------ | | |
| 213 | | Script | `deploy/azure/gate.sh` (laptop and CI run the SAME one) | | |
| 214 | | Workflow | `.forgejo/workflows/azure-gate.yml` (dispatch + nightly janitor) | | |
| 215 | | Identity | SP `mitosis-azure-gate`, Contributor on the subscription | | |
| 216 | | Secrets | `AZURE_GATE_{CLIENT_ID,CLIENT_SECRET,TENANT_ID,SUBSCRIPTION_ID}`, `AZURE_GATE_CLAUDE_TOKEN` (crew credential for the E2E; absent = loud SKIP) | | |
| 217 | | Cost | ~$0.30–0.60/run (RG-per-run, tagged; Cost Analysis groups by `run`) | | |
| 218 | ||
| 219 | Dispatch-only (each run is real money); `keep=true` + your `ssh_pubkey` turns a | |
| 220 | run into a live deployment you can use. Isolation: everything lives in | |
| 221 | `mitosis-gate-<run>`; the janitor reaps any gate RG whose `expires` tag passed, | |
| 222 | so crashed runs and forgotten keeps die within a day. |
scripts/coldstart-assert.sh +47 −7
| @@ -16,7 +16,8 @@ | ||
| 16 | 16 | # |
| 17 | 17 | # Requires: bash, kubectl, curl, jq; docker for A5 (must reach the k3d node |
| 18 | 18 | # container, i.e. run on the machine that hosts the cluster). bash 3.2 |
| 19 | # compatible (macOS). | |
| 19 | # compatible (macOS). On a bare-VM k3s ground (deploy/azure gate), run it ON | |
| 20 | # the node with NODE_MODE=local — A5 probes the host directly, no docker. | |
| 20 | 21 | # |
| 21 | 22 | # Assertions (each prints PASS/FAIL/SKIP + one evidence line): |
| 22 | 23 | # A1 all Flux Kustomizations Ready=True (>=10, incl. kyverno + kyverno-policies); |
| @@ -87,6 +88,35 @@ WEBHOOK_LOG_MIN_VERSION="0.7.6" # agents version that logs "system webhook verif | ||
| 87 | 88 | |
| 88 | 89 | k() { kubectl --context "" "$@"; } |
| 89 | 90 | |
| 91 | # Node-namespace probes (A5). Two grounds: | |
| 92 | # k3d (default): the node is a docker container named ${CTX}-server-0 — | |
| 93 | # docker exec reaches its netns and its crictl. | |
| 94 | # bare VM (NODE_MODE=local): this script runs ON the k3s node itself | |
| 95 | # (deploy/azure gate sshes in and runs it there) — no container exists; | |
| 96 | # the host's own shell IS the node namespace, and crictl is k3s's. | |
| 97 | NODE_MODE="" | |
| 98 | ||
| 99 | # GROUND — which substrate the platform germinated on. k3s (default): the full | |
| 100 | # recipe, every organ present. aks: a MANAGED Azure ground where | |
| 101 | # trim_gitops_aks removed the k3s-only organs (monitoring/grafana, node-fabric, | |
| 102 | # registry-node-config, coredns rewrites) — so A5 (the containerd-mirror probes) | |
| 103 | # is skipped and the Flux-Kustomization floor is lower. Everything else asserts | |
| 104 | # identically. A9-grafana already self-skips when the monitoring Kustomization | |
| 105 | # is absent, so it needs no guard here. | |
| 106 | GROUND="" | |
| 107 | KUST_MIN=10 | |
| 108 | [ "" = aks ] && KUST_MIN=6 # aks drops monitoring/node-fabric/registry-node-config/coredns | |
| 109 | node_sh() { | |
| 110 | if [ "" = local ]; then sh -c "$1"; else docker exec "" sh -c "$1"; fi | |
| 111 | } | |
| 112 | node_crictl() { | |
| 113 | if [ "" = local ]; then | |
| 114 | if command -v crictl >/dev/null 2>&1; then crictl "$@"; else k3s crictl "$@"; fi | |
| 115 | else | |
| 116 | docker exec "" crictl "$@" | |
| 117 | fi | |
| 118 | } | |
| 119 | ||
| 90 | 120 | T0= |
| 91 | 121 | FAIL_COUNT=0 |
| 92 | 122 | FAIL_LIST="" |
| @@ -385,7 +415,7 @@ while :; do | ||
| 385 | 415 | KUST_NOT_READY= |
| 386 | 416 | HAS_KYVERNO= |
| 387 | 417 | HAS_KYVERNO_POLICIES= |
| 388 | if [ "" -ge 10 ] && [ -z "" ] && | |
| 418 | if [ "" -ge "" ] && [ -z "" ] && | |
| 389 | 419 | [ -n "" ] && [ -n "" ]; then |
| 390 | 420 | break |
| 391 | 421 | fi |
| @@ -395,11 +425,11 @@ while :; do | ||
| 395 | 425 | sleep 10 |
| 396 | 426 | done |
| 397 | 427 | |
| 398 | if [ "" -ge 10 ] && [ -z "" ] && | |
| 428 | if [ "" -ge "" ] && [ -z "" ] && | |
| 399 | 429 | [ -n "" ] && [ -n "" ]; then |
| 400 | 430 | pass A1-kustomizations " Kustomizations all Ready=True, incl. kyverno + kyverno-policies: $(echo "" | awk '{print $1}' | tr '\n' ' ')" |
| 401 | 431 | else |
| 402 | fail A1-kustomizations "total= (want >=10) notReady=[] kyverno= kyverno-policies=" | |
| 432 | fail A1-kustomizations "total= (want >=) notReady=[] kyverno= kyverno-policies=" | |
| 403 | 433 | fi |
| 404 | 434 | |
| 405 | 435 | POL_COUNT= |
| @@ -509,6 +539,14 @@ else | ||
| 509 | 539 | fi |
| 510 | 540 | |
| 511 | 541 | # ---------- A5: registry mirror -------------------------------------------------- |
| 542 | # aks ground has no containerd mirror: trim_gitops_aks removes | |
| 543 | # registry-node-config, nodes pull from ACR + git.<domain> over public TLS, | |
| 544 | # and there is no reachable node namespace to probe. Skip all three sub-checks. | |
| 545 | if [ "" = aks ]; then | |
| 546 | skip A5-forwarder-running "aks ground: registry-node-config removed by trim_gitops_aks" | |
| 547 | skip A5-node-listener "aks ground: no containerd mirror / no reachable node namespace" | |
| 548 | skip A5-mirror-pull "aks ground: images come from ACR + git.<domain> public TLS, not the :31100 mirror" | |
| 549 | else | |
| 512 | 550 | |
| 513 | 551 | FWD_LINE= |
| 514 | 552 | if echo "" | grep -q 'Running'; then |
| @@ -518,7 +556,7 @@ else | ||
| 518 | 556 | fi |
| 519 | 557 | |
| 520 | 558 | NODE="-server-0" # k3d context k3d-X -> node container k3d-X-server-0 |
| 521 | LISTEN= | |
| 559 | LISTEN= | |
| 522 | 560 | if [ -n "" ]; then |
| 523 | 561 | pass A5-node-listener "node netns listener on :31100: " |
| 524 | 562 | else |
| @@ -532,14 +570,16 @@ fi | ||
| 532 | 570 | # fallback for old platforms where mcp may predate this convention. |
| 533 | 571 | PULL_REF= |
| 534 | 572 | case "" in "git."/*) ;; *) PULL_REF="git./plat/ci-builder:" ;; esac |
| 535 | if docker exec "" crictl pull "" >/dev/null 2>&1; then | |
| 573 | if node_crictl pull "" >/dev/null 2>&1; then | |
| 536 | 574 | pass A5-mirror-pull "crictl pull succeeded (anonymous)" |
| 537 | elif [ -n "" ] && docker exec "" crictl pull --creds ":" "" >/dev/null 2>&1; then | |
| 575 | elif [ -n "" ] && node_crictl pull --creds ":" "" >/dev/null 2>&1; then | |
| 538 | 576 | pass A5-mirror-pull "crictl pull succeeded (with forgejo-admin creds)" |
| 539 | 577 | else |
| 540 | 578 | fail A5-mirror-pull "crictl pull failed through the 127.0.0.1:31100 mirror" |
| 541 | 579 | fi |
| 542 | 580 | |
| 581 | fi # end GROUND != aks (A5) | |
| 582 | ||
| 543 | 583 | # ---------- A6: dispatcher + mcp Available --------------------------------------- |
| 544 | 584 | |
| 545 | 585 | AG_AVAIL= |
new above
offline