Public: anyone can read this; nobody can change it.
deploy/azure: mode=vms|aks in main.bicep, HA servers carve-out, AKS germinate profile #139
Check · Review · Land done · Live — Landed; nothing deploys from this branch, so this is the end of its path.
azure-aks-mode → main 10 commits
9 files
changed +1637
−38
bin/germinate +44 −1
| @@ -47,6 +47,26 @@ fi | ||
| 47 | 47 | # daughter forks mitosis from when the seed's parent is gone or unreachable. |
| 48 | 48 | # Overridable so an enterprise fleet can point at its own root forge. |
| 49 | 49 | : "${MITOSIS_FORK_ROOT:=https://git.open-platform.sh/plat/mitosis}" |
| 50 | # CLUSTER_PROFILE — the GROUND the daughter grows on. k3s (default): bare VMs / | |
| 51 | # k3d, node access assumed (registries.yaml mirror, ctr image side-load). | |
| 52 | # aks: a managed Azure cluster deployed by deploy/azure/main.bicep -p mode=aks — | |
| 53 | # no node access; ci-runner is built server-side with `az acr build`, the k3s | |
| 54 | # node couplings (monitoring/grafana, node-fabric, registry-node-config, | |
| 55 | # coredns rewrites) are trimmed by trim_gitops_aks, and Azure Monitor is the | |
| 56 | # observability surface. Requires ACR=<login server> (from the bicep outputs). | |
| 57 | : "${CLUSTER_PROFILE:=k3s}" | |
| 58 | if [ "$CLUSTER_PROFILE" = aks ]; then | |
| 59 | : "${ACR:?CLUSTER_PROFILE=aks requires ACR=<acr login server, e.g. mitosisabc.azurecr.io>}" | |
| 60 | : "${AKS_RG:?CLUSTER_PROFILE=aks requires AKS_RG=<resource group of the mode=aks deployment>}" | |
| 61 | command -v az >/dev/null 2>&1 || { echo "FATAL: CLUSTER_PROFILE=aks requires the az CLI (ci-runner builds via 'az acr build')"; exit 1; } | |
| 62 | # The ACME solver swap (trim_gitops_aks 4d/4e) needs the bicep-provisioned | |
| 63 | # identity + zone facts; derive them once here so the operator passes only RG. | |
| 64 | : "${AKS_SUBSCRIPTION_ID:=$(az account show --query id -o tsv)}" | |
| 65 | : "${AKS_DNS_ZONE:=$DOMAIN}" | |
| 66 | : "${AKS_CERT_MANAGER_CLIENT_ID:=$(az identity show -g "$AKS_RG" -n mitosis-cert-manager --query clientId -o tsv 2>/dev/null || true)}" | |
| 67 | [ -n "$AKS_CERT_MANAGER_CLIENT_ID" ] || echo "WARN: mitosis-cert-manager identity not found in $AKS_RG — LE solver stays on cloudflare and certs WILL fail" | |
| 68 | export AKS_SUBSCRIPTION_ID AKS_DNS_ZONE AKS_CERT_MANAGER_CLIENT_ID AKS_RG | |
| 69 | fi | |
| 50 | 70 | # BYO claude credentials (default empty) — sealed into plat-agents-secrets so a |
| 51 | 71 | # daughter launched with one can build apps the instant it reconciles. Captured |
| 52 | 72 | # into named locals so we can report (NOT echo) whether agents are credentialed, |
| @@ -124,6 +144,14 @@ GITOPS_DIR="$WORK/r-gitops" | ||
| 124 | 144 | [ "$PROFILE" = lean ] && trim_gitops_lean "$GITOPS_DIR" |
| 125 | 145 | strip_gitops_tenants "$GITOPS_DIR" # fresh daughter has no external orgs (any profile) |
| 126 | 146 | |
| 147 | # aks ground: strip the k3s node couplings + Grafana observability (Azure | |
| 148 | # Monitor replaces it) and repoint the runner image at ACR. Runs AFTER the | |
| 149 | # lean trim (it edits the already-trimmed tree) and BEFORE identity rewrite. | |
| 150 | if [ "$CLUSTER_PROFILE" = aks ]; then | |
| 151 | STEP="trim gitops for the aks ground" | |
| 152 | trim_gitops_aks "$GITOPS_DIR" "$ACR" | |
| 153 | fi | |
| 154 | ||
| 127 | 155 | # OLD-VINTAGE ONLY (dind seeds; current seeds ship no docker-daemon-config and |
| 128 | 156 | # skip this whole block): pin dind's bridge MTU to the cluster overlay so image |
| 129 | 157 | # builds' large downloads aren't silently dropped (k3d/flannel overlay = 1450; |
| @@ -732,7 +760,22 @@ CIRTAG=$(grep -ohE 'ci-runner:[0-9]+\.[0-9]+\.[0-9]+' \ | ||
| 732 | 760 | "$GITOPS_DIR"/clusters/local/platform/forgejo/*.yaml 2>/dev/null | head -1 | sed 's/.*://' || true) |
| 733 | 761 | : "${CIRTAG:=0.1.0}" |
| 734 | 762 | CIR_IMAGE="git.$DOMAIN/plat/ci-runner:$CIRTAG" |
| 735 | if [ ! -d "$WORK/r-ci-runner" ]; then | |
| 763 | if [ "$CLUSTER_PROFILE" = aks ]; then | |
| 764 | # Managed ground: no node runtime to side-load into. `az acr build` builds | |
| 765 | # server-side in ACR (no local docker) and the ScaledJob already points at | |
| 766 | # $ACR/plat/ci-runner (trim_gitops_aks rewrote it); kubelet pulls via the | |
| 767 | # AcrPull role the bicep granted — the chicken-and-egg never exists here. | |
| 768 | CIR_IMAGE="$ACR/plat/ci-runner:$CIRTAG" | |
| 769 | if [ ! -d "$WORK/r-ci-runner" ]; then | |
| 770 | echo " WARN: seed carries no plat/ci-runner — the isolated pool cannot start and" | |
| 771 | echo " NO CI will run. Re-cut the seed from a mitosis that bundles it." | |
| 772 | else | |
| 773 | echo "==> building $CIR_IMAGE server-side (az acr build; managed ground has no node to side-load)" | |
| 774 | az acr build --registry "${ACR%%.azurecr.io*}" --image "plat/ci-runner:$CIRTAG" "$WORK/r-ci-runner" >/dev/null \ | |
| 775 | && echo " $CIR_IMAGE built and pushed in ACR" \ | |
| 776 | || { echo "FATAL: az acr build failed — no runner image means no CI at all"; exit 1; } | |
| 777 | fi | |
| 778 | elif [ ! -d "$WORK/r-ci-runner" ]; then | |
| 736 | 779 | echo " WARN: seed carries no plat/ci-runner — the isolated pool cannot start and" |
| 737 | 780 | echo " NO CI will run. Re-cut the seed from a mitosis that bundles it." |
| 738 | 781 | elif ! command -v docker >/dev/null 2>&1; then |
bin/lib.sh +346
| @@ -351,6 +351,352 @@ verify_seed_bundles() { | ||
| 351 | 351 | echo " seed integrity: every bundle matches its manifest sha256" |
| 352 | 352 | } |
| 353 | 353 | |
| 354 | # trim_gitops_aks — adapt an (already lean-trimmed) gitops tree to a MANAGED | |
| 355 | # Azure ground (deploy/azure/main.bicep -p mode=aks). Called by germinate when | |
| 356 | # CLUSTER_PROFILE=aks, after trim_gitops_lean and before identity rewrite. | |
| 357 | # | |
| 358 | # What changes and why: | |
| 359 | # 1. monitoring/monitoring-scrapes GO — Azure Monitor (managed Prometheus + | |
| 360 | # Container Insights, enabled by the bicep) is the observability surface; | |
| 361 | # Grafana and its OAuth/SSO surface do not exist on this ground. | |
| 362 | # 2. node-fabric GOES — it exists to fix k3s flannel vxlan checksum offload; | |
| 363 | # AKS (azure CNI overlay + cilium) has no such datapath. | |
| 364 | # 3. registry-node-config GOES — the 127.0.0.1:31100 containerd realm-rewrite | |
| 365 | # exists because k3s nodes can't resolve .local registries; AKS nodes pull | |
| 366 | # git.<domain> over public DNS + trusted TLS, and infra images from ACR. | |
| 367 | # 4. coredns GOES — the k3s coredns-custom rewrite kept *.domain in-cluster; | |
| 368 | # on AKS the public zone resolves to the LB and hairpinning is fine. | |
| 369 | # 5. The isolated runner's ScaledJob image is repointed at ACR (germinate | |
| 370 | # builds it there with `az acr build`; kubelet pulls via AcrPull). | |
| 371 | # 6. A Kyverno MUTATE policy (aks-image-redirect) rewrites explicit external | |
| 372 | # registries (ghcr/quay/kyverno/k8s/forgejo/docker.io + the recipe's known | |
| 373 | # docker.io shortnames) to ACR at Pod admission — one policy instead of | |
| 374 | # per-chart values surgery. deploy/azure/mirror-images.sh MUST have run | |
| 375 | # first or redirected pods ErrImagePull. kube-system is excluded (AKS | |
| 376 | # managed addons are Microsoft's to pull from MCR). | |
| 377 | # 7. restrict-image-registries allowlist learns the ACR + mcr.microsoft.com. | |
| 378 | trim_gitops_aks() { # $1 = gitops clone $2 = ACR login server | |
| 379 | local g="$1" | |
| 380 | local acr="$2" | |
| 381 | local P="/clusters/local/platform" fk="/clusters/local/flux-kustomizations.yaml" | |
| 382 | local pk="/kustomization.yaml" | |
| 383 | ||
| 384 | # 1-4) strip the k3s-ground components from the platform Kustomization + tree. | |
| 385 | if [ -f "" ]; then | |
| 386 | awk -v strip="monitoring monitoring-scrapes node-fabric registry-node-config coredns" ' | |
| 387 | BEGIN { n = split(strip, a, " "); for (i = 1; i <= n; i++) drop[a[i]] = 1 } | |
| 388 | /^[[:space:]]*-[[:space:]]/ { | |
| 389 | item = | |
| 390 | sub(/^[[:space:]]*-[[:space:]]*/, "", item) | |
| 391 | sub(/\/.*$/, "", item) | |
| 392 | sub(/[[:space:]]+$/, "", item) | |
| 393 | if (item in drop) next | |
| 394 | } | |
| 395 | { print } | |
| 396 | ' "$pk" > "$pk.trim" && mv "$pk.trim" "$pk" | |
| 397 | fi | |
| 398 | rm -rf "/monitoring" "/monitoring-scrapes" "/node-fabric" "/registry-node-config" "/coredns" | |
| 399 | ||
| 400 | # monitoring ships as standalone Kustomization docs — chunk-drop any doc that | |
| 401 | # NAMES monitoring (also catches monitoring-scrapes and dependsOn entries; | |
| 402 | # anything depending on a stripped component cannot reconcile either). | |
| 403 | if [ -f "" ]; then | |
| 404 | awk ' | |
| 405 | function flush() { if (buf != "" && buf !~ /name:[[:space:]]*monitoring/) printf "%s", buf; buf = "" } | |
| 406 | /^---[[:space:]]*$/ { flush(); buf = "\n"; next } | |
| 407 | { buf = buf "\n" } | |
| 408 | END { flush() } | |
| 409 | ' "$fk" > "$fk.trim" && mv "$fk.trim" "$fk" | |
| 410 | fi | |
| 411 | ||
| 412 | # Grafana-coupled surfaces outside monitoring/: the agents flow console rides | |
| 413 | # grafana.<domain> + a forwardAuth against the Grafana session — no Grafana, | |
| 414 | # no surface (plat-flow CLI drives the same API; an Entra-fronted host is the | |
| 415 | # pilot follow-up). Presence-guarded: agents dir layouts vary by vintage. | |
| 416 | local af agdir="/agents" | |
| 417 | if [ -d "" ]; then | |
| 418 | for af in ; do | |
| 419 | echo " trim(aks): dropping grafana-coupled " | |
| 420 | local bn; bn= | |
| 421 | rm -f "" | |
| 422 | [ -f "/kustomization.yaml" ] && sed -i.bak "\||d" "/kustomization.yaml" && rm -f "/kustomization.yaml.bak" | |
| 423 | done | |
| 424 | fi | |
| 425 | ||
| 426 | # 4b) control-plane nodeSelectors GO — the k3s recipe pins fail-closed control | |
| 427 | # components (cert-manager, cnpg, mcp, kyverno, forgejo-pg) to the durable | |
| 428 | # server node; AKS nodes carry no control-plane label, so the pin makes | |
| 429 | # those pods permanently unschedulable (observed: first aks germination, | |
| 430 | # cert-manager Pending → whole platform chain stalled). Drops the | |
| 431 | # two-line `nodeSelector:` + control-plane pattern and bare control-plane | |
| 432 | # lines wherever they appear under platform/. | |
| 433 | local nsf | |
| 434 | for nsf in ; do | |
| 435 | awk ' | |
| 436 | /^[[:space:]]*nodeSelector:[[:space:]]*$/ { hold=; held=1; next } | |
| 437 | held && /node-role\.kubernetes\.io\/control-plane/ { held=0; next } | |
| 438 | held { print hold; held=0 } | |
| 439 | /node-role\.kubernetes\.io\/control-plane/ && !/^[[:space:]]*#/ { next } | |
| 440 | { print } | |
| 441 | END { if (held) print hold } | |
| 442 | ' "$nsf" > "$nsf.aks" && mv "$nsf.aks" "$nsf" | |
| 443 | echo " trim(aks): dropped control-plane pin in " | |
| 444 | done | |
| 445 | # …and the parents the strip can orphan: cert-manager's per-component keys | |
| 446 | # (webhook/cainjector/startupapicheck) exist in the recipe ONLY to carry the | |
| 447 | # pin — left empty they render null and nil-pointer the chart's templates | |
| 448 | # (observed: first aks germination, UpgradeFailed on | |
| 449 | # webhook-validating-webhook.yaml). POSIX classes: BSD sed has no \s. | |
| 450 | local cmf="/cert-manager/helmrelease.yaml" | |
| 451 | [ -f "" ] && sed -i.bak -E '/^[[:space:]]+(webhook|cainjector|startupapicheck):[[:space:]]*$/d' "" && rm -f ".bak" | |
| 452 | ||
| 453 | # 4c) explicit local-path storage classes GO — AKS has no such class; a PVC | |
| 454 | # naming it Pends forever and stalls the forgejo-pg handoff (observed: | |
| 455 | # first aks germination, "Cluster is unrecoverable"). Dropping the line | |
| 456 | # falls back to the cluster default (managed-csi — better: survives node | |
| 457 | # loss). Covers CNPG `storageClass:` and PVC `storageClassName:` forms. | |
| 458 | local spf | |
| 459 | for spf in ; do | |
| 460 | sed -i.bak -E '/storageClass(Name)?:[[:space:]]*local-path/d' "" && rm -f ".bak" | |
| 461 | echo " trim(aks): dropped local-path storage class in " | |
| 462 | done | |
| 463 | ||
| 464 | # 4d) ACME solver: cloudflare → azureDNS via workload identity. The bicep | |
| 465 | # ships a mitosis-cert-manager identity federated to the cert-manager | |
| 466 | # ServiceAccount with DNS Zone Contributor on the platform zone; the | |
| 467 | # germinate env supplies its clientID + subscription + resource group. | |
| 468 | # Without this swap the wildcard cert challenges fail ("no Cloudflare | |
| 469 | # credential") and CI cannot docker-login to git.<domain> over TLS. | |
| 470 | local liss="/letsencrypt/clusterissuer.yaml" | |
| 471 | if [ -f "" ] && grep -q cloudflare ""; then | |
| 472 | if [ -n "" ] && [ -n "" ] && [ -n "" ] && [ -n "" ]; then | |
| 473 | python3 - "" << | |
| 474 | import os, sys | |
| 475 | p = sys.argv[1] | |
| 476 | s = open(p).read() | |
| 477 | old = """ cloudflare: | |
| 478 | apiTokenSecretRef: | |
| 479 | name: cloudflare-dns-api | |
| 480 | key: api-token""" | |
| 481 | new = f""" # aks ground: Azure DNS via workload identity (no secret in-cluster) | |
| 482 | azureDNS: | |
| 483 | subscriptionID: {os.environ['AKS_SUBSCRIPTION_ID']} | |
| 484 | resourceGroupName: {os.environ['AKS_RG']} | |
| 485 | hostedZoneName: {os.environ['AKS_DNS_ZONE']} | |
| 486 | environment: AzurePublicCloud | |
| 487 | managedIdentity: | |
| 488 | clientID: {os.environ['AKS_CERT_MANAGER_CLIENT_ID']}""" | |
| 489 | if old in s: | |
| 490 | open(p, 'w').write(s.replace(old, new)) | |
| 491 | print(" trim(aks): LE solver cloudflare -> azureDNS") | |
| 492 | else: | |
| 493 | print(" WARN trim(aks): cloudflare solver block shape unexpected — swap it by hand") | |
| 494 | ||
| 495 | rm -f "/letsencrypt/cloudflare-dns-api.yaml" | |
| 496 | [ -f "/letsencrypt/kustomization.yaml" ] && sed -i.bak '/cloudflare-dns-api/d' "/letsencrypt/kustomization.yaml" && rm -f "/letsencrypt/kustomization.yaml.bak" | |
| 497 | else | |
| 498 | echo " WARN trim(aks): AKS_CERT_MANAGER_CLIENT_ID/AKS_SUBSCRIPTION_ID/AKS_RG/AKS_DNS_ZONE unset — LE solver left on cloudflare (certs WILL fail)" | |
| 499 | fi | |
| 500 | fi | |
| 501 | ||
| 502 | # 4e) cert-manager runs under that identity: annotate its ServiceAccount and | |
| 503 | # opt pods into the workload-identity webhook. | |
| 504 | if [ -f "" ] && [ -n "" ] && ! grep -q 'workload.identity' ""; then | |
| 505 | python3 - "" << | |
| 506 | import os, sys | |
| 507 | p = sys.argv[1] | |
| 508 | s = open(p).read() | |
| 509 | anchor = " values:\n" | |
| 510 | cid = os.environ['AKS_CERT_MANAGER_CLIENT_ID'] | |
| 511 | inject = f""" values: | |
| 512 | # aks ground: federate with the mitosis-cert-manager identity (DNS-01). | |
| 513 | serviceAccount: | |
| 514 | annotations: | |
| 515 | azure.workload.identity/client-id: {cid} | |
| 516 | podLabels: | |
| 517 | azure.workload.identity/use: "true" | |
| 518 | """ | |
| 519 | if anchor in s: | |
| 520 | open(p, 'w').write(s.replace(anchor, inject, 1)) | |
| 521 | print(" trim(aks): cert-manager wired to workload identity") | |
| 522 | else: | |
| 523 | print(" WARN trim(aks): cert-manager values anchor not found — wire workload identity by hand") | |
| 524 | ||
| 525 | fi | |
| 526 | ||
| 527 | # 4f) the git edge serves the PUBLIC (LE) certificate — AKS kubelets pull the | |
| 528 | # plat images (mcp, agents, apps) from git.<domain> over public TLS and | |
| 529 | # trust only public CAs; the plat-local cert x509-fails every pull | |
| 530 | # (observed: second aks germination, mcp ImagePullBackOff at the final | |
| 531 | # gate). The k3s ground keeps the internal cert (its nodes bypass via the | |
| 532 | # containerd mirror). | |
| 533 | local igf="/forgejo/ingress.yaml" | |
| 534 | if [ -f "" ] && grep -q 'secretName: wildcard-platform-tls' ""; then | |
| 535 | sed -i.bak 's/secretName: wildcard-platform-tls/secretName: wildcard-public-tls/' "" && rm -f ".bak" | |
| 536 | echo " trim(aks): git edge cert -> wildcard-public-tls (kubelet pulls need a public CA)" | |
| 537 | fi | |
| 538 | # …and that certificate must come from LE PRODUCTION: the recipe pins the | |
| 539 | # staging server (fine on k3s where the public cert is cosmetic), but staging | |
| 540 | # certs are not publicly trusted, so kubelet pulls x509-fail exactly like the | |
| 541 | # internal CA did (observed: second aks germination, "(STAGING) Artificial | |
| 542 | # Amaranth" on the edge). | |
| 543 | if [ -f "" ]; then | |
| 544 | sed -i.bak 's|https://acme-staging-v02.api.letsencrypt.org/directory|https://acme-v02.api.letsencrypt.org/directory|' "" && rm -f ".bak" | |
| 545 | fi | |
| 546 | ||
| 547 | # 4g) charts of k3s vintage HARDCODE storageClassName: local-path in PVC | |
| 548 | # templates (agents chart, app templates) — no values override exists. | |
| 549 | # A mutate policy strips it at admission so PVCs bind the cluster default | |
| 550 | # (managed-csi). Remove once the charts parameterize storage. | |
| 551 | if [ -d "/kyverno-policies" ]; then | |
| 552 | cat > "/kyverno-policies/aks-storage-default.yaml" << | |
| 553 | # aks ground only (written by trim_gitops_aks): strip hardcoded local-path | |
| 554 | # from PVCs at admission; AKS binds the default class (managed-csi) instead. | |
| 555 | apiVersion: kyverno.io/v1 | |
| 556 | kind: ClusterPolicy | |
| 557 | metadata: | |
| 558 | name: aks-storage-default | |
| 559 | labels: | |
| 560 | plat.sh/managed: "true" | |
| 561 | spec: | |
| 562 | background: false | |
| 563 | rules: | |
| 564 | - name: strip-local-path | |
| 565 | match: | |
| 566 | any: | |
| 567 | - resources: | |
| 568 | kinds: [PersistentVolumeClaim] | |
| 569 | preconditions: | |
| 570 | all: | |
| 571 | - key: '{{ request.object.spec.storageClassName || '''' }}' | |
| 572 | operator: Equals | |
| 573 | value: local-path | |
| 574 | mutate: | |
| 575 | patchesJson6902: |- | |
| 576 | - op: remove | |
| 577 | path: /spec/storageClassName | |
| 578 | ||
| 579 | if [ -f "/kyverno-policies/kustomization.yaml" ] && ! grep -q 'aks-storage-default' "/kyverno-policies/kustomization.yaml"; then | |
| 580 | sed -i.bak '/^resources:/a\ | |
| 581 | - aks-storage-default.yaml' "$P/kyverno-policies/kustomization.yaml" && rm -f "$P/kyverno-policies/kustomization.yaml.bak" | |
| 582 | fi | |
| 583 | fi | |
| 584 | # …but kyverno's webhooks skip SYSTEM namespaces (plat.sh/app selector), so | |
| 585 | # the policy covers tenant apps only. The agents chart lives in a system | |
| 586 | # namespace and hardcodes local-path too — strip it at Helm render time. | |
| 587 | local ahf="/agents/helmrelease.yaml" | |
| 588 | if [ -f "" ] && ! grep -q postRenderers ""; then | |
| 589 | python3 - "" << | |
| 590 | import sys | |
| 591 | p = sys.argv[1] | |
| 592 | s = open(p).read() | |
| 593 | anchor = "spec:\n" | |
| 594 | inject = """spec: | |
| 595 | # aks ground: the chart hardcodes storageClassName local-path in its PVC | |
| 596 | # templates and kyverno's webhooks skip system namespaces — strip the field | |
| 597 | # at render time so PVCs bind the cluster default (managed-csi). | |
| 598 | postRenderers: | |
| 599 | - kustomize: | |
| 600 | patches: | |
| 601 | - target: | |
| 602 | kind: PersistentVolumeClaim | |
| 603 | patch: | | |
| 604 | - op: remove | |
| 605 | path: /spec/storageClassName | |
| 606 | """ | |
| 607 | if anchor in s: | |
| 608 | i = s.index(anchor) | |
| 609 | open(p, 'w').write(s[:i] + inject + s[i+len(anchor):]) | |
| 610 | print(" trim(aks): agents postRenderer strips hardcoded local-path") | |
| 611 | else: | |
| 612 | print(" WARN trim(aks): agents helmrelease spec anchor not found — strip local-path by hand") | |
| 613 | ||
| 614 | fi | |
| 615 | ||
| 616 | # 5) runner image → ACR (domain-agnostic pattern: runs BEFORE identity rewrite, | |
| 617 | # so the tree still carries the parent's domain). | |
| 618 | local rf | |
| 619 | for rf in ; do | |
| 620 | sed -i.bak -E "s|git\.[^/\"']+/plat/ci-runner:|/plat/ci-runner:|g" "" && rm -f ".bak" | |
| 621 | echo " trim(aks): runner image -> /plat/ci-runner (in $(basename ""))" | |
| 622 | done | |
| 623 | ||
| 624 | # 6) admission-time ACR redirect for external images. One policy, no per-chart | |
| 625 | # values surgery; idempotent (regex misses leave the image untouched). | |
| 626 | local KP="/kyverno-policies" | |
| 627 | if [ -d "" ]; then | |
| 628 | cat > "/aks-image-redirect.yaml" << | |
| 629 | # aks ground only (written by trim_gitops_aks): every explicit external | |
| 630 | # registry the recipe pins is mirrored into ACR by deploy/azure/mirror-images.sh | |
| 631 | # (same path, host swapped) — this policy makes pods pull the mirror without | |
| 632 | # touching each chart's values. kube-system is Microsoft's (MCR-managed addons). | |
| 633 | apiVersion: kyverno.io/v1 | |
| 634 | kind: ClusterPolicy | |
| 635 | metadata: | |
| 636 | name: aks-image-redirect | |
| 637 | labels: | |
| 638 | plat.sh/managed: "true" | |
| 639 | spec: | |
| 640 | background: false | |
| 641 | rules: | |
| 642 | - name: redirect-containers | |
| 643 | match: | |
| 644 | any: | |
| 645 | - resources: | |
| 646 | kinds: [Pod] | |
| 647 | exclude: | |
| 648 | any: | |
| 649 | - resources: | |
| 650 | namespaces: [kube-system, kube-node-lease, kube-public, gatekeeper-system] | |
| 651 | mutate: | |
| 652 | foreach: | |
| 653 | - list: request.object.spec.containers | |
| 654 | patchStrategicMerge: | |
| 655 | spec: | |
| 656 | containers: | |
| 657 | - name: '{{ element.name }}' | |
| 658 | image: '{{ regex_replace_all(''^(ghcr\.io|quay\.io|reg\.kyverno\.io|registry\.k8s\.io|code\.forgejo\.org|docker\.io)/(.*)$'', regex_replace_all(''^(traefik|alpine|caddy|busybox)((:|@).*)?$'', regex_replace_all(''^(minio|alpine)/(.*)$'', element.image, ''docker.io/$1/$2''), ''docker.io/library/$1$2''), ''__ACR__/$2'') }}' | |
| 659 | - name: redirect-init-containers | |
| 660 | match: | |
| 661 | any: | |
| 662 | - resources: | |
| 663 | kinds: [Pod] | |
| 664 | # preconditions guard: only pods that HAVE initContainers | |
| 665 | preconditions: | |
| 666 | all: | |
| 667 | - key: '{{ request.object.spec.initContainers[] | length(@) }}' | |
| 668 | operator: GreaterThan | |
| 669 | value: 0 | |
| 670 | exclude: | |
| 671 | any: | |
| 672 | - resources: | |
| 673 | namespaces: [kube-system, kube-node-lease, kube-public, gatekeeper-system] | |
| 674 | mutate: | |
| 675 | foreach: | |
| 676 | - list: request.object.spec.initContainers | |
| 677 | patchStrategicMerge: | |
| 678 | spec: | |
| 679 | initContainers: | |
| 680 | - name: '{{ element.name }}' | |
| 681 | image: '{{ regex_replace_all(''^(ghcr\.io|quay\.io|reg\.kyverno\.io|registry\.k8s\.io|code\.forgejo\.org|docker\.io)/(.*)$'', regex_replace_all(''^(traefik|alpine|caddy|busybox)((:|@).*)?$'', regex_replace_all(''^(minio|alpine)/(.*)$'', element.image, ''docker.io/$1/$2''), ''docker.io/library/$1$2''), ''__ACR__/$2'') }}' | |
| 682 | ||
| 683 | sed -i.bak "s|__ACR__||g" "/aks-image-redirect.yaml" && rm -f "/aks-image-redirect.yaml.bak" | |
| 684 | if [ -f "/kustomization.yaml" ] && ! grep -q 'aks-image-redirect' "/kustomization.yaml"; then | |
| 685 | sed -i.bak '/^resources:/a\ | |
| 686 | - aks-image-redirect.yaml' "$KP/kustomization.yaml" && rm -f "$KP/kustomization.yaml.bak" | |
| 687 | fi | |
| 688 | # 7) the allowlist learns the mirror + Microsoft's registry (AKS addons). | |
| 689 | if [ -f "/restrict-image-registries.yaml" ] && ! grep -q "" "/restrict-image-registries.yaml"; then | |
| 690 | sed -i.bak -E "0,/^([[:space:]]*)- \"?git\./s||\1- \"/*\"\n\1- \"mcr.microsoft.com/*\"\n&|" \ | |
| 691 | "/restrict-image-registries.yaml" && rm -f "/restrict-image-registries.yaml.bak" \ | |
| 692 | || echo " WARN trim(aks): could not extend restrict-image-registries allowlist — add by hand" | |
| 693 | fi | |
| 694 | fi | |
| 695 | ||
| 696 | git -C "" add -A | |
| 697 | git -C "" -c user.email=mitosis@plat -c user.name=mitosis commit -q -m "mitosis: aks ground (Azure Monitor replaces grafana; strip node-fabric/registry-node-config/coredns; runner + external images via ACR)" || true | |
| 698 | } | |
| 699 | ||
| 354 | 700 | # publish_wiki — materialize the in-repo wiki/ source as the NATIVE Forgejo wiki |
| 355 | 701 | # of <owner>/mitosis. The wiki is documentation-as-an-organ: it ships inside the |
| 356 | 702 | # mitosis repo (reviewable, versioned with the code it documents), and every |
deploy/README.md +2
| @@ -46,6 +46,8 @@ official SSM parameter — no AMI table. | ||
| 46 | 46 | |
| 47 | 47 | ## Azure (Bicep) |
| 48 | 48 | |
| 49 | Full guide, both modes (bare VMs or AKS): [](azure/README.md). | |
| 50 | ||
| 49 | 51 | sh |
| 50 | 52 | az login # or a service principal |
| 51 | 53 | az group create -n mitosis -l eastus |
deploy/azure/README.md added +385
| @@ -0,0 +1,385 @@ | ||
| 1 | # Deploy Mitosis on Azure | |
| 2 | ||
| 3 | This directory deploys a Mitosis platform on Microsoft Azure. One Bicep | |
| 4 | template, `main.bicep`, does the deployment. The template has two modes. Set | |
| 5 | the `mode` parameter to select one. | |
| 6 | ||
| 7 | | Mode | What Azure runs | Who installs the platform | Select it when | | |
| 8 | | --------------- | ------------------------------------------------------------------- | ----------------------------------------------- | ----------------------------------------------------------------- | | |
| 9 | | `vms` (default) | 1 or 3 Ubuntu VMs with k3s | The leader VM, automatically, at its first boot | You want a sovereign platform on plain VMs. | | |
| 10 | | `aks` | A managed AKS cluster with ACR, Azure DNS, Key Vault, Azure Monitor | You, from your machine, after the deployment | You want an Azure-managed cluster and Azure-native services only. | | |
| 11 | ||
| 12 | The two modes install the same platform: Forgejo, Flux, MCP, CNPG, Kyverno, | |
| 13 | KEDA, MinIO, and the app model. The guide for all clouds is | |
| 14 | [](../README.md). Read it for the parts that are the same on | |
| 15 | AWS, Azure, and DigitalOcean. | |
| 16 | ||
| 17 | ## Words used in this document | |
| 18 | ||
| 19 | | Word | Meaning | | |
| 20 | | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | | |
| 21 | | Platform | The full Mitosis installation: git server, container registry, cluster services, and apps. | | |
| 22 | | Germination | The procedure that installs the platform into an empty cluster. `bin/germinate` runs it. | | |
| 23 | | Leader | The first VM in `vms` mode. It runs germination. | | |
| 24 | | Sovereign key | The `age` key that encrypts the secrets of the platform. There is no other copy. If you lose it, you lose the secrets. | | |
| 25 | | Resource group | The Azure container that holds all resources of one deployment. | | |
| 26 | | Domain | The DNS name of the platform, for example `plat.example.com`. Services get names below it: `git.`, `mcp.`, `www.`, and one name for each app. | | |
| 27 | | Seed | The archive `genesis/seed.tar.gz`. It holds the platform repositories that germination installs. | | |
| 28 | ||
| 29 | ## Before you start | |
| 30 | ||
| 31 | You need: | |
| 32 | ||
| 33 | - An Azure subscription and the `az` CLI. Log in with `az login`. | |
| 34 | - Permission to create resources in a resource group. | |
| 35 | - For `vms` mode: an SSH public key, for example `~/.ssh/id_ed25519.pub`. | |
| 36 | - For `aks` mode: `kubectl`, `flux`, `sops`, `age`, and `jq` on your machine, | |
| 37 | and a seed. | |
| 38 | - A domain that you control. In `vms` mode the domain is optional (see | |
| 39 | [](#zero-dns-vms-mode-only)). In `aks` mode the domain is required. | |
| 40 | - Optional: one Claude credential, so the build crew can start work | |
| 41 | immediately. Use `claudeCodeOauthToken` (subscription) or `anthropicApiKey` | |
| 42 | (API billing). Do not set both. | |
| 43 | ||
| 44 | Do these two steps one time on a new subscription: | |
| 45 | ||
| 46 | 1. Register the resource providers. Run | |
| 47 | `az provider register -n <name> --wait` for each name the mode uses: | |
| 48 | - `vms`: `Microsoft.Compute`, `Microsoft.Network` | |
| 49 | - `aks`: `Microsoft.ContainerService`, `Microsoft.ContainerRegistry`, | |
| 50 | `Microsoft.Network`, `Microsoft.OperationalInsights`, | |
| 51 | `Microsoft.KeyVault`, `Microsoft.ManagedIdentity` | |
| 52 | 2. Check the vCPU quota with `az vm list-usage -l <region> -o table`. New | |
| 53 | subscriptions get quota for the v6, v7, and B-series VM families only. | |
| 54 | The default VM sizes in this template are in these families. `vms` mode | |
| 55 | needs 4 free vCPUs for each VM. `aks` mode needs 8 free vCPUs. | |
| 56 | ||
| 57 | CAUTION: `main.bicepparam` is in `.gitignore` because it can hold | |
| 58 | credentials. Copy `main.bicepparam.example` and edit the copy. Do not commit | |
| 59 | it. | |
| 60 | ||
| 61 | ## Mode `vms` — sovereign k3s on VMs | |
| 62 | ||
| 63 | ### Resources this mode creates | |
| 64 | ||
| 65 | All resources go into the resource group that you name. Set `ha=true` to get | |
| 66 | three VMs. | |
| 67 | ||
| 68 | | Resource | Name | Details | | |
| 69 | | -------------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------ | | |
| 70 | | Network security group | `mitosis-nodes` | Opens TCP 22 (SSH, from `allowedSshCidr`) and TCP 80, 443, 2222 (from the Internet). | | |
| 71 | | Virtual network | `mitosis` | Address space `10.60.0.0/24`, one subnet `nodes`. | | |
| 72 | | Public IP address (1 or 3) | `mitosis-leader`, `mitosis-server-1`, `-2` | Standard SKU, static. The IP address does not change when the VM restarts. | | |
| 73 | | Network interface (1 or 3) | Same names as the public IPs | Static private IPs `10.60.0.10`, `.11`, `.12`. | | |
| 74 | | Virtual machine (1 or 3) | Same names as the public IPs | Ubuntu 24.04 LTS, `Standard_D4as_v7` (4 vCPU, 16 GB), 80 GB StandardSSD OS disk. | | |
| 75 | ||
| 76 | The VM does the rest at first boot. Cloud-init writes the registry recipe and | |
| 77 | the node contract from [](../lib/). Then a systemd unit installs k3s, | |
| 78 | clones Mitosis, and runs germination. Azure credentials never go to the VMs. | |
| 79 | ||
| 80 | ### Steps | |
| 81 | ||
| 82 | 1. Create the resource group. | |
| 83 | ||
| 84 | sh | |
| 85 | az group create -n mitosis -l eastus | |
| 86 | ||
| 87 | ||
| 88 | 2. Start the deployment. Replace the domain with yours. | |
| 89 | ||
| 90 | sh | |
| 91 | cd deploy/azure | |
| 92 | az deployment group create -g mitosis -f main.bicep \ | |
| 93 | -p domain=plat.example.com \ | |
| 94 | -p adminSshPublicKey="$(cat ~/.ssh/id_ed25519.pub)" | |
| 95 | ||
| 96 | ||
| 97 | Options: add `-p ha=true` for three servers. Add | |
| 98 | `-p vmSize=Standard_B4ms` for a cheaper, burstable VM. Add | |
| 99 | `-p allowedSshCidr=<your-ip>/32` to limit SSH access. | |
| 100 | ||
| 101 | 3. Read the outputs. They show the IP addresses, the DNS record, and the next | |
| 102 | commands. | |
| 103 | ||
| 104 | sh | |
| 105 | az deployment group show -g mitosis -n main --query properties.outputs | |
| 106 | ||
| 107 | ||
| 108 | 4. Create one DNS record at your DNS provider. Use the `dnsRecord` output: | |
| 109 | `*.plat.example.com A <leader-ip>`. In HA mode, put all three IPs in the | |
| 110 | record. This one record covers `git.`, `mcp.`, `www.`, and all apps. | |
| 111 | ||
| 112 | 5. Watch germination. It takes approximately 30 to 45 minutes. | |
| 113 | ||
| 114 | sh | |
| 115 | ssh mitosis@<leader-ip> journalctl -u mitosis-node -f | |
| 116 | ||
| 117 | ||
| 118 | 6. When germination is complete, read the platform card. | |
| 119 | ||
| 120 | sh | |
| 121 | ssh mitosis@<leader-ip> sudo cat /root/PLATFORM.md | |
| 122 | ||
| 123 | ||
| 124 | 7. Copy the sovereign key off the VM immediately. The card shows how. The key | |
| 125 | is at `/root/fork-<domain>.age` on the leader. | |
| 126 | ||
| 127 | WARNING: Keep the sovereign key in a safe place. It is the only key to the | |
| 128 | secrets of the platform. If you delete the VM and you do not have the key, | |
| 129 | the secrets are lost. | |
| 130 | ||
| 131 | The VM login user is `mitosis`, or the value of `adminUsername`. Azure VMs | |
| 132 | have no root login. Git over SSH listens on port 2222. | |
| 133 | ||
| 134 | ### Zero-DNS (`vms` mode only) | |
| 135 | ||
| 136 | If you do not have a domain, set `-p domain=auto`. At first boot, the leader | |
| 137 | reads its public IP and names the platform `<ip-with-dashes>.sslip.io`. This | |
| 138 | name resolves without any DNS record. The platform is available at | |
| 139 | `https://git.<that-name>` when germination is complete. | |
| 140 | ||
| 141 | ### Add servers to a live platform | |
| 142 | ||
| 143 | `main.bicep` with `ha=true` is for new deployments only. To add two servers | |
| 144 | to a platform that already runs, use `servers.bicep`. It creates only the | |
| 145 | joiner VMs (`mitosis-server-1`, `-2`) in the existing network. | |
| 146 | ||
| 147 | 1. Make sure the leader runs embedded etcd. On a single-node install, write | |
| 148 | `cluster-init: true` to `/etc/rancher/k3s/config.yaml` on the leader and | |
| 149 | restart k3s. k3s migrates from sqlite to etcd in place. | |
| 150 | 2. Read the join token from the leader: | |
| 151 | `ssh mitosis@<leader-ip> sudo cat /var/lib/rancher/k3s/server/token`. | |
| 152 | 3. Deploy the joiners. | |
| 153 | ||
| 154 | sh | |
| 155 | az deployment group create -g mitosis -f servers.bicep \ | |
| 156 | -p domain=plat.example.com \ | |
| 157 | -p adminSshPublicKey="$(cat ~/.ssh/id_ed25519.pub)" \ | |
| 158 | -p k3sToken="<token-from-step-2>" | |
| 159 | ||
| 160 | ||
| 161 | 4. Wait 5 to 10 minutes. Then check: `kubectl get nodes` shows 3 nodes | |
| 162 | `Ready`. | |
| 163 | 5. Add the two new IPs to the `*.<domain>` A record. | |
| 164 | ||
| 165 | NOTE: `k3sVersion` in `servers.bicep` must match the version the leader runs | |
| 166 | (`k3s --version`). | |
| 167 | ||
| 168 | ## Mode `aks` — Azure-managed cluster | |
| 169 | ||
| 170 | In this mode Azure runs the cluster. There is no cloud-init and no node | |
| 171 | contract. You run germination from your machine against the cluster | |
| 172 | kubeconfig, after the deployment. | |
| 173 | ||
| 174 | ### Resources this mode creates | |
| 175 | ||
| 176 | | Resource | Name | Details | | |
| 177 | | --------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | | |
| 178 | | Log Analytics workspace | `mitosis-logs` | Container logs. Retention 30 days. | | |
| 179 | | Container registry (ACR) | `mitosis<unique>` (or `acrName`) | Standard SKU. Holds the mirrored infra images and the CI runner image. | | |
| 180 | | AKS cluster | `mitosis-aks` | 2 × `Standard_D4ds_v6` (4 vCPU, 16 GB), zones 2 and 3, Azure CNI overlay with Cilium, workload identity, OIDC issuer. | | |
| 181 | | Role assignment | AcrPull on the ACR | Lets the cluster nodes pull images from the ACR. | | |
| 182 | | Public DNS zone | `<domain>` | Created when `createDnsZone=true` (default). Delegate the domain to it. | | |
| 183 | | Managed identity + federated credential | `mitosis-cert-manager` | Lets cert-manager answer DNS-01 challenges in the zone (DNS Zone Contributor). Gives public TLS certificates. | | |
| 184 | | Key Vault | `mkv<unique>` (or `keyVaultName`) | Standard SKU, RBAC, soft delete on. Holds the sops key at pilot hardening. | | |
| 185 | | Managed identity + federated credential | `mitosis-flux-sops` | Lets the Flux kustomize-controller decrypt sops secrets with a Key Vault key (Key Vault Crypto User). | | |
| 186 | | Azure Monitor | Container Insights + managed Prometheus | Enabled on the cluster. Replaces the Grafana of the k3s recipe. | | |
| 187 | ||
| 188 | Azure creates one more resource at germination: a Standard Load Balancer | |
| 189 | with a public IP for the Traefik Service. It serves ports 80, 443, and 2222. | |
| 190 | It lives in the node resource group (`MC_...`). | |
| 191 | ||
| 192 | ### Steps | |
| 193 | ||
| 194 | 1. Create the resource group and start the deployment. | |
| 195 | ||
| 196 | sh | |
| 197 | az group create -n mitosis-aks -l eastus | |
| 198 | cd deploy/azure | |
| 199 | az deployment group create -g mitosis-aks -f main.bicep \ | |
| 200 | -p domain=plat.example.com -p mode=aks | |
| 201 | ||
| 202 | ||
| 203 | 2. Read the outputs: `acrLoginServer`, `dnsNameServers`, `keyVaultName`, | |
| 204 | `certManagerClientId`, `fluxSopsClientId`. | |
| 205 | ||
| 206 | sh | |
| 207 | az deployment group show -g mitosis-aks -n main --query properties.outputs | |
| 208 | ||
| 209 | ||
| 210 | 3. Delegate the domain. At your registrar or DNS host, add NS records for | |
| 211 | `plat.example.com` that point to the four `dnsNameServers`. | |
| 212 | ||
| 213 | 4. Mirror the infra images into the ACR. The copy runs on the Azure side and | |
| 214 | takes approximately 2 minutes. Do this before germination. Pods that are | |
| 215 | redirected to the ACR cannot start if the image is not there. | |
| 216 | ||
| 217 | sh | |
| 218 | deploy/azure/mirror-images.sh <acrLoginServer> | |
| 219 | ||
| 220 | ||
| 221 | 5. Get the kubeconfig. | |
| 222 | ||
| 223 | sh | |
| 224 | az aks get-credentials -g mitosis-aks -n mitosis-aks | |
| 225 | ||
| 226 | ||
| 227 | 6. Start germination from the repository root. Germination creates the | |
| 228 | sovereign key on your machine. It needs the `az` CLI: it builds the CI | |
| 229 | runner image with `az acr build` and reads the cert-manager identity from | |
| 230 | the resource group. | |
| 231 | ||
| 232 | sh | |
| 233 | SEED=genesis/seed.tar.gz CLUSTER_PROFILE=aks \ | |
| 234 | ACR=<acrLoginServer> AKS_RG=mitosis-aks DOMAIN=plat.example.com \ | |
| 235 | FORK_KEY_ACK=1 bin/germinate | |
| 236 | ||
| 237 | ||
| 238 | 7. Create the zone records as soon as the Traefik load balancer has a public | |
| 239 | IP. This happens a few minutes after germination starts. Watch with | |
| 240 | `kubectl get svc -n traefik traefik -w`. | |
| 241 | ||
| 242 | sh | |
| 243 | az network dns record-set a add-record -g mitosis-aks -z plat.example.com -n '*' -a <lb-ip> --ttl 60 | |
| 244 | az network dns record-set a add-record -g mitosis-aks -z plat.example.com -n '@' -a <lb-ip> --ttl 60 | |
| 245 | ||
| 246 | ||
| 247 | CAUTION: The CI of the platform pushes to `git.<domain>` over public DNS. | |
| 248 | Chart builds fail until these records exist. Use `--ttl 60` while you | |
| 249 | make changes. A wrong cached record delays you by one full build. | |
| 250 | ||
| 251 | 8. Back up the sovereign key immediately. | |
| 252 | ||
| 253 | 9. Verify: run `scripts/coldstart-assert.sh`. Then build the first app from | |
| 254 | `_app-template` and open `https://<app>.<domain>`. The certificate is | |
| 255 | public. | |
| 256 | ||
| 257 | ### Optional: Entra ID sign-in on Forgejo | |
| 258 | ||
| 259 | Run this after germination. You need rights to create app registrations in | |
| 260 | the tenant. | |
| 261 | ||
| 262 | sh | |
| 263 | KUBECONFIG=~/.kube/config deploy/azure/entra-forgejo-oidc.sh plat.example.com | |
| 264 | ||
| 265 | ||
| 266 | The script creates an Entra app registration, a client secret (2 years), and | |
| 267 | an OpenID Connect sign-in source named `entra` in Forgejo. Users then see | |
| 268 | "Sign in with entra" on `git.<domain>`. Local accounts continue to work. | |
| 269 | ||
| 270 | ### What is different from `vms` mode | |
| 271 | ||
| 272 | - Storage is the AKS default `managed-csi` (Azure Disk). Volumes survive node | |
| 273 | loss. Germination removes the `local-path` storage class from the recipe. | |
| 274 | A Kyverno policy removes it from tenant PVCs at admission. | |
| 275 | - TLS is public. cert-manager gets certificates from Let's Encrypt | |
| 276 | (production) with DNS-01 in the Azure DNS zone, through the | |
| 277 | `mitosis-cert-manager` identity. The git edge serves this certificate, so | |
| 278 | the cluster nodes can pull platform images over public TLS. | |
| 279 | - Germination removes the control-plane `nodeSelector` pins of the k3s | |
| 280 | recipe. AKS nodes do not have that label. | |
| 281 | - The CI runner image is built on the Azure side with `az acr build`. There | |
| 282 | is no image side-loading. | |
| 283 | - A Kyverno policy, `aks-image-redirect`, points external registries to the | |
| 284 | ACR at Pod admission. Nodes pull only from the ACR and from the platform | |
| 285 | registry. | |
| 286 | - Grafana, node-fabric, registry-node-config, and the CoreDNS rewrites are | |
| 287 | removed at germination. Use **Insights** on the AKS resource for logs and | |
| 288 | metrics. | |
| 289 | ||
| 290 | ### Known limits (proof of concept) | |
| 291 | ||
| 292 | - The Grafana checks in `coldstart-assert.sh` do not apply. Skip them. | |
| 293 | - Images with an explicit registry, and the known docker.io short names | |
| 294 | (traefik, alpine, caddy, busybox, minio), are redirected to the ACR. Other | |
| 295 | docker.io short names are not. | |
| 296 | - The flow console (agents operator UI) has no hostname in this mode. The | |
| 297 | `kit/plat-flow` CLI works. | |
| 298 | - CNPG clusters keep their `ghcr.io` image name in the spec. `aks-image-redirect` | |
| 299 | changes only the Pod. CNPG then sees a difference and restarts the tenant | |
| 300 | database. Fix: extend the policy to the CNPG `Cluster` resource. | |
| 301 | - CNPG PodDisruptionBudgets block AKS node operations (scale, upgrade). Drain | |
| 302 | the node by hand with `--disable-eviction` first. | |
| 303 | - Some VM sizes attach few disks (`D4as_v7`: 12). Count the volumes before you | |
| 304 | change the VM size. | |
| 305 | ||
| 306 | The record of the first AKS platform, with costs, commands, and lessons, is | |
| 307 | in [](aks-cost-experiment.md). | |
| 308 | ||
| 309 | ### Pilot hardening (before customer secrets) | |
| 310 | ||
| 311 | 1. sops → Key Vault: add an `azure_keyvault` recipient in `.sops.yaml`. Flux | |
| 312 | decrypts through the `mitosis-flux-sops` identity. Then remove the age key | |
| 313 | from the cluster. Keep the offline copy for emergencies. | |
| 314 | 2. Enable the agents module with the customer's Anthropic credential. Give | |
| 315 | the flow console its own hostname and Entra authentication. | |
| 316 | 3. Mirror the app base images (bun, node, postgres, ...) into the ACR. Extend | |
| 317 | `aks-image-redirect`. Then set `restrict-image-registries` to Enforce. | |
| 318 | 4. Private cluster, App Gateway + WAF, Azure Policy, as the customer requires. | |
| 319 | 5. Send CNPG backups to Azure Blob instead of MinIO. | |
| 320 | ||
| 321 | ## Parameters | |
| 322 | ||
| 323 | | Parameter | Mode | Default | Meaning | | |
| 324 | | ----------------------------------------- | ----- | ---------------------- | --------------------------------------------------------------------- | | |
| 325 | | `domain` | both | (required) | The domain of the platform. `auto` is allowed in `vms` mode only. | | |
| 326 | | `mode` | both | `vms` | `vms` or `aks`. | | |
| 327 | | `location` | both | Resource group region | Azure region for all resources. | | |
| 328 | | `ha` | `vms` | `false` | `true` = three servers with embedded etcd. | | |
| 329 | | `adminSshPublicKey` | `vms` | (required) | Your SSH public key. | | |
| 330 | | `adminUsername` | `vms` | `mitosis` | VM login user. | | |
| 331 | | `vmSize` | `vms` | `Standard_D4as_v7` | 4 vCPU / 8 GB is the minimum. | | |
| 332 | | `diskGb` | `vms` | `80` | OS disk size. 80 GB is the minimum. | | |
| 333 | | `allowedSshCidr` | `vms` | `*` | Source range for SSH. | | |
| 334 | | `claudeCodeOauthToken`, `anthropicApiKey` | `vms` | empty | Set one. Goes to the leader only. In `aks` mode, set it at germination. | | |
| 335 | | `k3sToken` | `vms` | new GUID | Pass your own to keep re-deployments from replacing the VMs. | | |
| 336 | | `k3sVersion` | `vms` | `v1.31.5+k3s1` | k3s release. | | |
| 337 | | `mitosisUrl`, `mitosisRef` | `vms` | `plat/mitosis`, `main` | Where the nodes clone Mitosis from. | | |
| 338 | | `aksNodeCount` | `aks` | `2` | Nodes in the system pool. | | |
| 339 | | `aksVmSize` | `aks` | `Standard_D4ds_v6` | Node size. | | |
| 340 | | `aksZones` | `aks` | `['2', '3']` | Availability zones. `[]` = non-zonal. | | |
| 341 | | `aksVersion` | `aks` | empty | Kubernetes version. Empty = the AKS default. | | |
| 342 | | `createDnsZone` | `aks` | `true` | `false` = use a zone that you already have. | | |
| 343 | | `acrName`, `keyVaultName` | `aks` | derived | Globally unique names. | | |
| 344 | ||
| 345 | ## Outputs | |
| 346 | ||
| 347 | | Output | Mode | Meaning | | |
| 348 | | --------------------------------------------------------- | ----- | ------------------------------------------------------------ | | |
| 349 | | `nodeIps`, `dnsRecord`, `nextSteps` | `vms` | The VM IPs, the one DNS record to create, the next commands. | | |
| 350 | | `acrLoginServer`, `aksName`, `oidcIssuer` | `aks` | The ACR host, the cluster name, the OIDC issuer URL. | | |
| 351 | | `dnsNameServers` | `aks` | The four name servers to delegate the domain to. | | |
| 352 | | `keyVaultName`, `certManagerClientId`, `fluxSopsClientId` | `aks` | Client IDs for the sops and cert-manager identities. | | |
| 353 | ||
| 354 | ## Cost (approximate, eastus, retail) | |
| 355 | ||
| 356 | | Mode | Compute | Other | | |
| 357 | | ------------------- | ------------------------------------- | ----------------------------------------------------- | | |
| 358 | | `vms`, 1 VM | `Standard_D4as_v7` ≈ $125 / month | OS disk, public IP. | | |
| 359 | | `vms`, 1 VM, budget | `Standard_B4ms` ≈ $60 / month | Burstable. Slower builds. | | |
| 360 | | `vms`, HA | 3 × the single-VM price | | | |
| 361 | | `aks`, default | 2 × `Standard_D4ds_v6` ≈ $364 / month | ACR, Standard Load Balancer, Log Analytics ingestion. | | |
| 362 | ||
| 363 | ## Remove the deployment | |
| 364 | ||
| 365 | sh | |
| 366 | az group delete -n <resource-group> | |
| 367 | ||
| 368 | ||
| 369 | WARNING: The secrets of the platform are removed with the cluster. Keep the | |
| 370 | sovereign key if you keep any backups. | |
| 371 | ||
| 372 | NOTE (`aks` mode): the Key Vault has soft delete on. Purge it before you | |
| 373 | create a new vault with the same name: | |
| 374 | `az keyvault purge -n <keyVaultName>`. | |
| 375 | ||
| 376 | ## Files in this directory | |
| 377 | ||
| 378 | | File | Purpose | | |
| 379 | | ------------------------- | ----------------------------------------------------------- | | |
| 380 | | `main.bicep` | The deployment template. Both modes. | | |
| 381 | | `main.bicepparam.example` | Example parameter file. Copy, edit, do not commit the copy. | | |
| 382 | | `servers.bicep` | Add two k3s servers to a live `vms` platform. | | |
| 383 | | `mirror-images.sh` | Copy the infra images into the ACR (`aks` mode). | | |
| 384 | | `entra-forgejo-oidc.sh` | Add Entra ID sign-in to Forgejo (`aks` mode). | | |
| 385 | | `aks-cost-experiment.md` | Record of the first AKS platform: costs, commands, lessons. | |
deploy/azure/aks-cost-experiment.md added +268
| @@ -0,0 +1,268 @@ | ||
| 1 | # AKS cost-down experiment — mitosis-aks (git.aks.benjivers.com) | |
| 2 | ||
| 3 | > **2026-08-18, end of day: TORN DOWN.** Ben ordered full teardown of both Azure | |
| 4 | > mitosis platforms (RGs `mitosis` + `mitosis-aks`, cold, no data export). | |
| 5 | > This file stays as the record of what the experiment found — the constraints, | |
| 6 | > gotchas, and az commands below apply to any future re-germinate on Azure. | |
| 7 | ||
| 8 | Started 2026-08-18. Goal: strip the AKS POC to one VM and reduce CPU/memory cost. | |
| 9 | Every change below is logged with its exact rollback command. Steps are executed | |
| 10 | top to bottom; status is updated as each one lands. | |
| 11 | ||
| 12 | Cluster: `mitosis-aks` in RG `mitosis-aks`, eastus, AKS 1.35.6, Free tier. | |
| 13 | kubectl context: `aks-mitosis`. | |
| 14 | ||
| 15 | --- | |
| 16 | ||
| 17 | ## Baseline (before any change) — captured 2026-08-18 | |
| 18 | ||
| 19 | - Node pool `system`: **2× Standard_D4ds_v6** (4 vCPU / 16 GB, ephemeral OS disk 80 GB), | |
| 20 | zones [2,3], autoscaler OFF, mode System. | |
| 21 | - Retail price D4ds_v6 eastus: **$0.249/hr** → 2 nodes ≈ **$364/mo** compute. | |
| 22 | - Live usage: CPU ~370m + ~250m (6–9%); memory 6.2 GB + 5.6 GB (~50%/node). | |
| 23 | - Requests: 2246m+2747m CPU (58%/71%), 3930Mi+5142Mi memory (33%/43%). | |
| 24 | - Monitoring: Container Insights (omsagent) **enabled** → Log Analytics workspace | |
| 25 | `/subscriptions/1d655bbf-4fc9-48dd-bf75-2d3f72bb64a1/resourceGroups/mitosis-aks/providers/Microsoft.OperationalInsights/workspaces/mitosis-logs`; | |
| 26 | managed Prometheus (azureMonitorMetrics) **enabled**. | |
| 27 | The ama-* pods request ~770m CPU / ~2.3 GB memory total, plus ingestion billing. | |
| 28 | - Health: `https://git.aks.benjivers.com/api/healthz` → 200 (~0.16 s). | |
| 29 | - All pod anti-affinity is `preferred` (kyverno, coredns, metrics-server, | |
| 30 | konnectivity, cilium-operator) → nothing hard-blocks single-node scheduling. | |
| 31 | - PDBs: all minAvailable=1, satisfiable on one node. | |
| 32 | - Workloads present: forgejo (+ cnpg forgejo-pg), agents, mcp, minio, traefik, | |
| 33 | sablier, registry-mirror, flux, kyverno, keda, cert-manager, cnpg-system, | |
| 34 | backup cronjobs, tenant app `comed-fleet--www--pr-4` (www-pg cnpg cluster). | |
| 35 | ||
| 36 | Single-node feasibility math (requests, daemonsets counted once): | |
| 37 | - As-is: ~4.34 cores requested vs ~3.86 allocatable → **does not fit**. | |
| 38 | - Minus monitoring (~770m): ~3.58 cores → fits. Memory ~5.4 GB requested → fits easily. | |
| 39 | ||
| 40 | Bicep note: `deploy/azure/main.bicep` pins `aksVmSize = 'Standard_D4ds_v6'`, | |
| 41 | `aksNodeCount` default (2, "POC floor"), and wires omsagent → `mitosis-logs` | |
| 42 | (line ~336). A re-germinate will restore all of today's baseline unless the | |
| 43 | bicep is updated to match whatever end-state we keep. | |
| 44 | ||
| 45 | --- | |
| 46 | ||
| 47 | ## Step 1 — disable Container Insights (omsagent addon) | |
| 48 | ||
| 49 | Removes ama-logs daemonset + ama-logs-rs (~340m CPU / ~860Mi requests) and stops | |
| 50 | Log Analytics ingestion billing. | |
| 51 | ||
| 52 | sh | |
| 53 | az aks disable-addons -g mitosis-aks -n mitosis-aks -a monitoring | |
| 54 | ||
| 55 | ||
| 56 | **Rollback:** | |
| 57 | sh | |
| 58 | az aks enable-addons -g mitosis-aks -n mitosis-aks -a monitoring \ | |
| 59 | --workspace-resource-id /subscriptions/1d655bbf-4fc9-48dd-bf75-2d3f72bb64a1/resourceGroups/mitosis-aks/providers/Microsoft.OperationalInsights/workspaces/mitosis-logs | |
| 60 | ||
| 61 | (The workspace `mitosis-logs` itself is NOT deleted by this step, so rollback is lossless | |
| 62 | going forward; only the gap window has no logs.) | |
| 63 | ||
| 64 | Status: **DONE 2026-08-18** — omsagent.enabled=false, all ama-logs pods gone. | |
| 65 | ||
| 66 | ## Step 2 — disable managed Prometheus (azureMonitorMetrics) | |
| 67 | ||
| 68 | Removes ama-metrics deployments + ama-metrics-node daemonset (~425m CPU / ~1.4Gi requests). | |
| 69 | Grafana-on-Azure dashboards fed by this stop updating (in-cluster metrics-server is | |
| 70 | unaffected; `kubectl top` keeps working). | |
| 71 | ||
| 72 | sh | |
| 73 | az aks update -g mitosis-aks -n mitosis-aks --disable-azure-monitor-metrics | |
| 74 | ||
| 75 | ||
| 76 | **Rollback:** | |
| 77 | sh | |
| 78 | az aks update -g mitosis-aks -n mitosis-aks --enable-azure-monitor-metrics | |
| 79 | ||
| 80 | ||
| 81 | Status: **DONE 2026-08-18** — metrics.enabled=false. Azure printed a warning that it | |
| 82 | deletes `azmonitoring.coreos.com` custom resources (PodMonitor/ServiceMonitor CRs from | |
| 83 | the managed addon); rollback recreates the defaults automatically. | |
| 84 | ||
| 85 | ## Step 3 — scale node pool 2 → 1 | |
| 86 | ||
| 87 | sh | |
| 88 | az aks nodepool scale -g mitosis-aks --cluster-name mitosis-aks -n system --node-count 1 | |
| 89 | ||
| 90 | ||
| 91 | Expected: VMSS removes one node; pods reschedule onto the survivor. Duplicate | |
| 92 | replicas (coredns, metrics-server, konnectivity, kyverno-admission ×3, | |
| 93 | cilium-operator, azure-wi-webhook) co-locate — anti-affinity is preferred-only. | |
| 94 | Brief blips while forgejo / traefik reschedule if they were on the removed node. | |
| 95 | ||
| 96 | **Rollback:** | |
| 97 | sh | |
| 98 | az aks nodepool scale -g mitosis-aks --cluster-name mitosis-aks -n system --node-count 2 | |
| 99 | ||
| 100 | ||
| 101 | **Verify after:** all pods Running (no Pending), `curl https://git.aks.benjivers.com/api/healthz` | |
| 102 | → 200, `flux get kustomizations` all Ready, tenant app pods healthy. | |
| 103 | ||
| 104 | Status: PENDING | |
| 105 | ||
| 106 | ## Step 4 — cheaper VM size (EXECUTED 2026-08-18, landed on D4as_v7) | |
| 107 | ||
| 108 | VM size cannot be changed in place; blue/green nodepool swap required: | |
| 109 | ||
| 110 | | SKU | eastus $/hr | ~$/mo (1 node) | notes | | |
| 111 | |---|---|---|---| | |
| 112 | | Standard_D4ds_v6 (current) | 0.249 | 182 | local SSD → ephemeral OS disk (no disk cost) | | |
| 113 | | Standard_D4as_v6 | 0.182 | 133 | AMD, no local disk → needs managed OS disk (+~$10/mo) | | |
| 114 | | Standard_B4as_v2 | 0.150 | 110 | burstable 4vCPU/16GB; fine for this idle profile; managed OS disk | | |
| 115 | | Standard_D2ds_v6 | 0.125 | 91 | 2vCPU/8GB — memory too small at current usage (~10GB) | | |
| 116 | ||
| 117 | Procedure (when we do it): | |
| 118 | sh | |
| 119 | # add new System-mode pool | |
| 120 | az aks nodepool add -g mitosis-aks --cluster-name mitosis-aks -n system2 \ | |
| 121 | --mode System --node-count 1 --node-vm-size Standard_B4as_v2 | |
| 122 | # drain + delete old pool | |
| 123 | kubectl --context aks-mitosis cordon -l agentpool=system | |
| 124 | kubectl --context aks-mitosis drain -l agentpool=system --ignore-daemonsets --delete-emptydir-data | |
| 125 | az aks nodepool delete -g mitosis-aks --cluster-name mitosis-aks -n system | |
| 126 | ||
| 127 | **Rollback:** re-add a `system` pool with the original size | |
| 128 | (`--node-vm-size Standard_D4ds_v6 --node-osdisk-type Ephemeral --node-osdisk-size 80 --zones 2 3`) | |
| 129 | and delete `system2`. Use the `az rest` form below if the CLI crashes. | |
| 130 | ||
| 131 | Status: **DONE 2026-08-18** — but NOT with B4as_v2. What actually happened: | |
| 132 | ||
| 133 | - **Quota**: subscription has 0 vCPU quota for ALL B-v2 families (Basv2/Bsv2/Bpsv2) | |
| 134 | in eastus → B4as_v2 impossible without a quota request. Old BSFamily (B4ms, | |
| 135 | $0.166/hr) has 10 free vCPUs but B4ms sustained baseline is only ~0.9 cores — | |
| 136 | platform idles at 1.5–1.8 cores → would throttle. Rejected. | |
| 137 | - **Chosen**: Standard_D4as_v7 ($0.182/hr ≈ $133/mo), StandardDasv7Family had 6 | |
| 138 | free vCPUs. Full-performance, AMD, no local disk → Managed OS disk 64 GB. | |
| 139 | - **az CLI bug**: `az aks nodepool add` crashes client-side | |
| 140 | (`AttributeError: 'AgentPool' object has no attribute 'artifact_streaming_profile'`, | |
| 141 | azure-cli SDK-model mismatch) even with --no-wait. Workaround — raw ARM call: | |
| 142 | sh | |
| 143 | az rest --method put --url "https://management.azure.com/subscriptions/1d655bbf-4fc9-48dd-bf75-2d3f72bb64a1/resourceGroups/mitosis-aks/providers/Microsoft.ContainerService/managedClusters/mitosis-aks/agentPools/system2?api-version=2025-05-01" \ | |
| 144 | --body '{"properties":{"mode":"System","count":1,"vmSize":"Standard_D4as_v7","osDiskSizeGB":64,"osDiskType":"Managed","osType":"Linux","availabilityZones":["2"]}}' | |
| 145 | ||
| 146 | (`az aks nodepool scale/show/list/delete` still work; only add is broken. Delete | |
| 147 | old pool likewise done via `az rest --method delete`.) | |
| 148 | - **Zone safety**: all PVs are StandardSSD_ZRS with multi-zone nodeAffinity, so | |
| 149 | the new pool's zone didn't matter. Verified before the swap. | |
| 150 | - Swap procedure used: create system2 → wait Ready → `kubectl cordon` + | |
| 151 | `drain --disable-eviction` old node (PDB bypass again) → `az rest` DELETE old | |
| 152 | `system` pool. | |
| 153 | - **Transient after swap**: platform app images pull from git.aks.benjivers.com's | |
| 154 | own Forgejo registry; while forgejo relocated nodes the registry 503'd → | |
| 155 | ImagePullBackOff on agents/tenant pods for ~3 min, self-healed once forgejo was up. | |
| 156 | Chicken-and-egg to remember for any single-node move. | |
| 157 | - **NEW HARD CONSTRAINT — disk attach limit**: D4as_v7 attaches max **12** Azure | |
| 158 | disks (old D4ds_v6: 16). Cluster already has **18** disk PVs; it only works | |
| 159 | because hibernated PR apps keep their disks detached. At >12 concurrently-awake | |
| 160 | disk consumers, pods will stick Pending "exceed max volume count". Mitigations: | |
| 161 | move small volumes to azurefile (no attach limit), keep PR apps hibernated, or | |
| 162 | go back to a Ddsv6-family SKU. | |
| 163 | ||
| 164 | ## Step 5 (planned) — trim workload requests via gitops | |
| 165 | ||
| 166 | Biggest non-system requests: agents 500m/1Gi, forgejo 250m/512Mi + pg 100m/512Mi, | |
| 167 | traefik 200m/150Mi, kyverno-admission 3×100m (replicas → 1 saves 200m/256Mi). | |
| 168 | These are Flux/Helm-managed — change in the gitops repo, not kubectl, or Flux | |
| 169 | reverts it. Do after Steps 1–3 prove stable. | |
| 170 | ||
| 171 | Status: NOT STARTED | |
| 172 | ||
| 173 | --- | |
| 174 | ||
| 175 | ## Execution log | |
| 176 | ||
| 177 | - 2026-08-18: Step 1 done. `az aks disable-addons -a monitoring` → omsagent disabled, 0 ama pods left. | |
| 178 | - 2026-08-18: Step 2 done. `--disable-azure-monitor-metrics` → managed Prometheus off. | |
| 179 | Combined CPU requests fell 5.0 → 4.0 cores (1820m + 2167m across the 2 nodes). | |
| 180 | - 2026-08-18: Step 3 attempt 1 **FAILED**: plain `az aks nodepool scale --node-count 1` | |
| 181 | errored (pool provisioningState=Failed, node auto-uncordoned). Root cause: AKS drain | |
| 182 | respects PDBs, and every CNPG cluster (forgejo-pg + each comed-fleet tenant pg) has | |
| 183 | minAvailable=1 with 1 replica → 0 allowed disruptions → eviction blocked → timeout. | |
| 184 | **Lesson: any future node op (scale/upgrade/resize) on this cluster needs a manual | |
| 185 | force-drain first.** | |
| 186 | - 2026-08-18: Fix applied: | |
| 187 | `kubectl cordon aks-system-40081872-vmss000001` | |
| 188 | `kubectl drain aks-system-40081872-vmss000001 --ignore-daemonsets --delete-emptydir-data --disable-eviction --grace-period=30 --timeout=180s` | |
| 189 | (`--disable-eviction` deletes pods directly, bypassing PDBs; CNPG rebuilt each pg pod | |
| 190 | on node 000000 within ~1 min. git.aks.benjivers.com healthz went 503 for ~1 min while | |
| 191 | forgejo-pg restarted, back to 200 after.) | |
| 192 | - 2026-08-18: `az aks nodepool delete-machines --machine-names ...vmss000001` REJECTED | |
| 193 | (InvalidParameter: won't delete "all" system nodes — it ignores the cordoned node when | |
| 194 | counting). Not usable here; plain scale-in works instead (VMSS default scale-in policy | |
| 195 | removes the highest instance ID = the drained 000001). | |
| 196 | - 2026-08-18: Step 3 attempt 2 — re-ran `az aks nodepool scale --node-count 1` after the | |
| 197 | manual drain; in progress. | |
| 198 | - Note: comed-fleet tenant apps (design/extract/invoices/m5bridge/www, prod + PRs) were | |
| 199 | actively deploying during the scale-down — pg pod churn in those namespaces is a mix of | |
| 200 | CNPG re-init and live fleet deploys, not all caused by us. | |
| 201 | - 2026-08-18: Step 3 **DONE** — pool at 1 node (vmss000000), provisioningState Succeeded, | |
| 202 | healthz 200. Forgejo took ~1 min of 503 during the pg force-drain, recovered on its own. | |
| 203 | - 2026-08-18: **Post-scale incident**: node CPU requests hit 3827m (99%) and | |
| 204 | comed-fleet--www--prod/www-pg-1 stuck FailedScheduling "Insufficient cpu" (CNPG kept | |
| 205 | deleting/recreating the Pending pod — that's what the 0s-age pod churn was). Actual CPU | |
| 206 | *usage* was only ~46%; this is request over-provisioning, not real load. | |
| 207 | Quick fix applied (kubectl, NOT in git — see revert-risk note): | |
| 208 | `kubectl scale deploy -n kyverno kyverno-admission-controller --replicas=1` (frees 200m) | |
| 209 | `kubectl scale deploy -n kube-system metrics-server --replicas=1` (frees 156m) | |
| 210 | → www-pg-1 scheduled and Running; requests down to 3727m (96%). | |
| 211 | **Revert-risk**: kyverno replica count is Helm/Flux-managed — reverts on next | |
| 212 | HelmRelease upgrade; metrics-server is an AKS managed addon — the addon-manager may | |
| 213 | restore replicas=2 at any time. Make durable via gitops (Step 5) if single-node stays. | |
| 214 | Rollback of the quick fix: same commands with `--replicas=3` / `--replicas=2`. | |
| 215 | - Known-benign on 1 node: second cilium-operator replica Pending (hostPort conflict — | |
| 216 | only one node to bind on). Clears itself if we ever scale back to 2. | |
| 217 | - 2026-08-18: **Churn classification** — the constant comed-fleet pg pod recreation is NOT | |
| 218 | from the scale-down: CNPG shows every tenant cluster in "Primary instance is being | |
| 219 | restarted without a switchover" / UpgradingInstance loops, and those clusters are only | |
| 220 | 14–42 min old. The agent fleet was actively deploying/upgrading comed-fleet apps | |
| 221 | throughout the experiment; each deploy rolls its pg. forgejo-pg: healthy. | |
| 222 | Pre-existed the node change (seen in the very first pod snapshot). | |
| 223 | ||
| 224 | ## End state (2026-08-18, after Step 4) | |
| 225 | ||
| 226 | - **1× Standard_D4as_v7** (pool `system2`, zone 2, 64 GB managed OS disk), monitoring | |
| 227 | addons off → **~$133/mo compute + ~$5/mo OS disk + $0 ingestion** | |
| 228 | (baseline was ~$364/mo + Log Analytics ingestion → **~62% cut**). | |
| 229 | - healthz 200, forgejo + core platform healthy, backups green (nightly dumps Complete, | |
| 230 | weekly restore-drill scheduled). | |
| 231 | - Node: CPU requests ~99% booked (usage far lower). Memory fine. Disk-attach ceiling | |
| 232 | 12 (18 PVs exist; OK only while PR apps stay hibernated). | |
| 233 | ||
| 234 | ## Client-readiness findings (2026-08-18 probe) | |
| 235 | ||
| 236 | 1. **CRITICAL — CNPG restart loop on every tenant DB**: kyverno `aks-image-redirect` | |
| 237 | mutates pod images to ACR, but CNPG Cluster spec still says ghcr.io → CNPG sees | |
| 238 | spec≠pod, restarts primary forever ("Primary instance is being restarted without a | |
| 239 | switchover" on all 9 comed-fleet clusters; tenant apps 503 with no ready DB). | |
| 240 | forgejo-pg is immune (spec already ACR). Fix: extend aks-image-redirect to also | |
| 241 | mutate `clusters.postgresql.cnpg.io` spec.imageName (keeps sovereignty policy | |
| 242 | intact) — one-off `kubectl patch` of each Cluster's imageName to the ACR name | |
| 243 | proves it (patch attempt was permission-blocked this session; needs Ben). | |
| 244 | 2. **Tenant TLS is the internal CA**: tenant ingresses use `wildcard-plat-tls` | |
| 245 | (issuer plat-local-ca) → browsers reject; forgejo/mcp use the public LE | |
| 246 | `wildcard-public-tls` (valid to 2026-11-16). Fix: tenant ingress template should | |
| 247 | reference the public wildcard (reflect/copy it into tenant namespaces). | |
| 248 | 3. **CPU requests 99%** — no headroom for new tenants; needs Step 5 request trim. | |
| 249 | 4. **Disk attach ceiling** — see Step 4 note (12 < 18 PVs). | |
| 250 | 5. **CNPG PDBs block every AKS node op** (scale/upgrade) — manual | |
| 251 | `drain --disable-eviction` required each time; platform-level fix desirable. | |
| 252 | 6. **No monitoring/alerting now** — we turned off both Azure layers; client-ready | |
| 253 | needs at least cheap in-cluster alerting or re-enable metrics only. | |
| 254 | 7. **bicep drift** — main.bicep still pins 2×D4ds_v6 + omsagent; re-germinate | |
| 255 | resurrects the $364/mo baseline. Update aksVmSize/aksNodeCount/omsagent. | |
| 256 | 8. az CLI `aks nodepool add` broken locally — use the az rest form (Step 4). | |
| 257 | ||
| 258 | ## Full restore to baseline (if it all goes wrong) | |
| 259 | ||
| 260 | sh | |
| 261 | az aks nodepool scale -g mitosis-aks --cluster-name mitosis-aks -n system --node-count 2 | |
| 262 | az aks enable-addons -g mitosis-aks -n mitosis-aks -a monitoring \ | |
| 263 | --workspace-resource-id /subscriptions/1d655bbf-4fc9-48dd-bf75-2d3f72bb64a1/resourceGroups/mitosis-aks/providers/Microsoft.OperationalInsights/workspaces/mitosis-logs | |
| 264 | az aks update -g mitosis-aks -n mitosis-aks --enable-azure-monitor-metrics | |
| 265 | kubectl --context aks-mitosis scale deploy -n kyverno kyverno-admission-controller --replicas=3 | |
| 266 | kubectl --context aks-mitosis scale deploy -n kube-system metrics-server --replicas=2 | |
| 267 | ||
| 268 | (order doesn't matter except: scale nodes FIRST if pods are stuck Pending) |
deploy/azure/entra-forgejo-oidc.sh added +54
| @@ -0,0 +1,54 @@ | ||
| 1 | #!/usr/bin/env bash | |
| 2 | # entra-forgejo-oidc.sh — wire Entra ID sign-in into a germinated aks-ground | |
| 3 | # Forgejo. Run AFTER germination, by an operator with rights to create app | |
| 4 | # registrations in the tenant (customer ops runs this in their own tenant). | |
| 5 | # | |
| 6 | # deploy/azure/entra-forgejo-oidc.sh <domain> [<display-name>] | |
| 7 | # | |
| 8 | # Creates (idempotently) an Entra app registration with the Forgejo OIDC | |
| 9 | # redirect URI, a client secret (2y), and registers a generic OpenID Connect | |
| 10 | # authentication source in Forgejo via the admin API. Users then sign in with | |
| 11 | # "Entra ID" on git.<domain>; local accounts keep working (break-glass admin). | |
| 12 | set -euo pipefail | |
| 13 | ||
| 14 | DOMAIN="" | |
| 15 | NAME="" | |
| 16 | REDIRECT="https://git./user/oauth2/entra/callback" | |
| 17 | TENANT= | |
| 18 | ||
| 19 | APP_ID= | |
| 20 | if [ -z "" ] || [ "" = "null" ]; then | |
| 21 | echo "==> creating Entra app registration ''" | |
| 22 | APP_ID= | |
| 23 | --web-redirect-uris "" \ | |
| 24 | --enable-id-token-issuance true \ | |
| 25 | --query appId -o tsv) | |
| 26 | else | |
| 27 | echo "==> reusing Entra app '' (); ensuring redirect URI" | |
| 28 | az ad app update --id "" --web-redirect-uris "" | |
| 29 | fi | |
| 30 | ||
| 31 | echo "==> minting client secret (2 years)" | |
| 32 | SECRET= | |
| 33 | ||
| 34 | # Forgejo admin API needs the platform admin credential — read it from the | |
| 35 | # cluster the way the operator card documents (never echoed). | |
| 36 | : "" | |
| 37 | ADMIN_PW= | |
| 38 | ||
| 39 | DISCOVERY="https://login.microsoftonline.com//v2.0/.well-known/openid-configuration" | |
| 40 | echo "==> registering OIDC auth source 'entra' in Forgejo (discovery: tenant )" | |
| 41 | # The stable interface across Forgejo versions is the admin CLI inside the pod | |
| 42 | # (the REST path for auth sources has moved between releases). | |
| 43 | POD= | |
| 44 | kubectl exec -n forgejo "" -- forgejo admin auth add-oauth \ | |
| 45 | --name entra --provider openidConnect \ | |
| 46 | --key "" --secret "" \ | |
| 47 | --auto-discover-url "" \ | |
| 48 | 2>/dev/null \ | |
| 49 | || kubectl exec -n forgejo "" -- forgejo admin auth update-oauth \ | |
| 50 | --id "$(kubectl exec -n forgejo "" -- forgejo admin auth list 2>/dev/null | awk '/entra/{print $1; exit}')" \ | |
| 51 | --key "" --secret "" --auto-discover-url "" | |
| 52 | ||
| 53 | echo "==> done. 'Sign in with entra' is live on https://git." | |
| 54 | : "" # reserved: REST fallback for Forgejo versions without the auth CLI |
deploy/azure/main.bicep +275 −37
| @@ -1,62 +1,109 @@ | ||
| 1 | // Mitosis on Azure — bare VMs, one deployment. | |
| 1 | // Mitosis on Azure — one deployment, two grounds. | |
| 2 | 2 | // |
| 3 | 3 | // az group create -n mitosis -l eastus |
| 4 | // | |
| 5 | // VMs (sovereign k3s, the default): | |
| 4 | 6 | // az deployment group create -g mitosis -f main.bicep \ |
| 5 | 7 | // -p domain=plat.example.com -p adminSshPublicKey="$(cat ~/.ssh/id_ed25519.pub)" |
| 6 | 8 | // |
| 7 | // One VM by default; -p ha=true grows three k3s servers (embedded etcd). | |
| 8 | // The leader germinates the platform; every node is born with the registry | |
| 9 | // recipe and its own copy of the CI runner image. See ../README.md. | |
| 9 | // AKS (Azure-pure: managed cluster + ACR + Azure DNS + Log Analytics + Key Vault): | |
| 10 | // az deployment group create -g mitosis-aks -f main.bicep \ | |
| 11 | // -p domain=plat.example.com -p mode=aks | |
| 10 | 12 | // |
| 11 | // The node contract (env file + bootstrap script + unit + registry recipe) is | |
| 12 | // defined ONCE in deploy/lib + recipe/ — this file renders the same template | |
| 13 | // Terraform does, with replace() instead of templatefile(). | |
| 13 | // mode=vms: one VM by default; -p ha=true grows three k3s servers (embedded | |
| 14 | // etcd). The leader germinates the platform; every node is born with the | |
| 15 | // registry recipe and its own copy of the CI runner image. See ../README.md. | |
| 16 | // | |
| 17 | // mode=aks: a managed cluster the operator germinates AFTERWARDS with | |
| 18 | // CLUSTER_PROFILE=aks bin/germinate (no cloud-init role — there are no nodes | |
| 19 | // to bootstrap). Ships ACR (infra images + ci-runner via `az acr build`), | |
| 20 | // a public Azure DNS zone for the domain (delegate NS at your registrar), | |
| 21 | // Log Analytics + managed Prometheus (Azure Monitor replaces the k3s | |
| 22 | // recipe's Grafana), Key Vault (sops key custody), and workload identities | |
| 23 | // for cert-manager (DNS-01) and Flux (Key Vault sops decryption). | |
| 24 | // | |
| 25 | // The VM node contract (env file + bootstrap script + unit + registry recipe) | |
| 26 | // is defined ONCE in deploy/lib + recipe/ — this file renders the same | |
| 27 | // template Terraform does, with replace() instead of templatefile(). | |
| 14 | 28 | |
| 15 | 29 | @description('The platform\'s sovereign domain (you own its DNS), e.g. plat.example.com.') |
| 16 | 30 | param domain string |
| 17 | 31 | |
| 18 | @description('false = one VM. true = three k3s servers with embedded etcd (survives one node loss).') | |
| 32 | @description('vms = self-managed k3s on bare VMs (sovereign default). aks = Azure-pure managed cluster (AKS + ACR + Azure DNS + Log Analytics + Key Vault); germinate with CLUSTER_PROFILE=aks after deploy.') | |
| 33 | @allowed(['vms', 'aks']) | |
| 34 | param mode string = 'vms' | |
| 35 | ||
| 36 | // ── vms-mode parameters ────────────────────────────────────────────────────── | |
| 37 | ||
| 38 | @description('vms: false = one VM. true = three k3s servers with embedded etcd (survives one node loss).') | |
| 19 | 39 | param ha bool = false |
| 20 | 40 | |
| 21 | @description('Your SSH public key (contents of ~/.ssh/id_ed25519.pub) — how you reach the VMs.') | |
| 22 | param adminSshPublicKey string | |
| 41 | @description('Your SSH public key (contents of ~/.ssh/id_ed25519.pub) — how you reach the VMs. REQUIRED for mode=vms; unused for mode=aks.') | |
| 42 | param adminSshPublicKey string = '' | |
| 23 | 43 | |
| 24 | 44 | @description('VM login user (Azure images have no root login).') |
| 25 | 45 | param adminUsername string = 'mitosis' |
| 26 | 46 | |
| 27 | @description('4 vCPU / 8 GB is the floor (germination builds several images). D4as_v7 = 4 vCPU / 16 GB and matches the quota fresh subscriptions actually get (v6/v7 + B-series; check az vm list-usage). B4ms is the burstable budget option; D4as_v5 for older subscriptions.') | |
| 47 | @description('vms: 4 vCPU / 8 GB is the floor (germination builds several images). D4as_v7 = 4 vCPU / 16 GB and matches the quota fresh subscriptions actually get (v6/v7 + B-series; check az vm list-usage). B4ms is the burstable budget option; D4as_v5 for older subscriptions.') | |
| 28 | 48 | param vmSize string = 'Standard_D4as_v7' |
| 29 | 49 | |
| 30 | @description('OS disk size. Images + Postgres + the platform registry live here; 80 GB is a comfortable floor.') | |
| 50 | @description('vms: OS disk size. Images + Postgres + the platform registry live here; 80 GB is a comfortable floor.') | |
| 31 | 51 | param diskGb int = 80 |
| 32 | 52 | |
| 33 | @description('Source allowed to SSH (port 22). Tighten to your IP, e.g. 203.0.113.7/32.') | |
| 53 | @description('vms: source allowed to SSH (port 22). Tighten to your IP, e.g. 203.0.113.7/32.') | |
| 34 | 54 | param allowedSshCidr string = '*' |
| 35 | 55 | |
| 36 | @description('Claude subscription credential for the build crew. Set exactly one of this or anthropicApiKey (or neither — the crew idles until you seal one later).') | |
| 56 | @description('Claude subscription credential for the build crew. Set exactly one of this or anthropicApiKey (or neither — the crew idles until you seal one later). vms mode only; on aks the credential is sealed at germination.') | |
| 37 | 57 | @secure() |
| 38 | 58 | param claudeCodeOauthToken string = '' |
| 39 | 59 | |
| 40 | @description('API-billing credential for the build crew (alternative to claudeCodeOauthToken).') | |
| 60 | @description('API-billing credential for the build crew (alternative to claudeCodeOauthToken). vms mode only.') | |
| 41 | 61 | @secure() |
| 42 | 62 | param anthropicApiKey string = '' |
| 43 | 63 | |
| 44 | @description('k3s cluster join token. Auto-generated per deployment — pass your own to keep re-deployments from replacing the VMs.') | |
| 64 | @description('k3s cluster join token. Auto-generated per deployment — pass your own to keep re-deployments from replacing the VMs. vms mode only.') | |
| 45 | 65 | @secure() |
| 46 | 66 | param k3sToken string = newGuid() |
| 47 | 67 | |
| 48 | @description('k3s release the nodes install — matches the version the platform is validated on.') | |
| 68 | @description('k3s release the nodes install — matches the version the platform is validated on. vms mode only.') | |
| 49 | 69 | param k3sVersion string = 'v1.31.5+k3s1' |
| 50 | 70 | |
| 51 | @description('Where nodes clone mitosis from. Point at your own platform\'s mitosis repo to grow a child of YOUR platform.') | |
| 71 | @description('Where nodes clone mitosis from. Point at your own platform\'s mitosis repo to grow a child of YOUR platform. vms mode only (aks germination clones on the operator host).') | |
| 52 | 72 | param mitosisUrl string = 'https://git.open-platform.sh/plat/mitosis' |
| 53 | 73 | |
| 54 | @description('Branch or tag of mitosis to deploy.') | |
| 74 | @description('Branch or tag of mitosis to deploy. vms mode only.') | |
| 55 | 75 | param mitosisRef string = 'main' |
| 56 | 76 | |
| 77 | // ── aks-mode parameters ────────────────────────────────────────────────────── | |
| 78 | ||
| 79 | @description('aks: node count for the (single, System-mode) pool. 2 is the POC floor; germination CI builds want the headroom.') | |
| 80 | param aksNodeCount int = 2 | |
| 81 | ||
| 82 | @description('aks: node size. Pick a family with free regional quota (az vm list-usage); D4ds_v6 = 4 vCPU / 16 GB + temp disk.') | |
| 83 | param aksVmSize string = 'Standard_D4ds_v6' | |
| 84 | ||
| 85 | @description('aks: availability zones for the node pool. Empty array = non-zonal (regions without zones).') | |
| 86 | param aksZones array = ['2', '3'] | |
| 87 | ||
| 88 | @description('aks: Kubernetes version. Empty = the AKS default for the region.') | |
| 89 | param aksVersion string = '' | |
| 90 | ||
| 91 | @description('aks: create a public Azure DNS zone for the domain (delegate NS records at your registrar). false = bring your own zone.') | |
| 92 | param createDnsZone bool = true | |
| 93 | ||
| 94 | @description('aks: ACR name (globally unique, alphanumeric). Default derives from the resource group.') | |
| 95 | param acrName string = 'mitosis${uniqueString(resourceGroup().id)}' | |
| 96 | ||
| 97 | @description('aks: Key Vault name (globally unique, 3-24 chars). Default derives from the resource group.') | |
| 98 | param keyVaultName string = 'mkv${uniqueString(resourceGroup().id)}' | |
| 99 | ||
| 57 | 100 | param location string = resourceGroup().location |
| 58 | 101 | |
| 59 | var nodeCount = ha ? 3 : 1 | |
| 102 | // ═════════════════════════════════ vms mode ═════════════════════════════════ | |
| 103 | ||
| 104 | // nodeCount 0 in aks mode empties every vms-mode loop — same template, no drift | |
| 105 | // for existing VM deployments (mode defaults to vms; resources render identically). | |
| 106 | var nodeCount = mode == 'vms' ? (ha ? 3 : 1) : 0 | |
| 60 | 107 | // 10.60/24 is clear of k3s's 10.42/16 (pods) + 10.43/16 (services). Static |
| 61 | 108 | // private IPs (.10/.11/.12): every node knows the leader with no discovery. |
| 62 | 109 | var subnetCidr = '10.60.0.0/24' |
| @@ -77,7 +124,7 @@ var cloudInitTmpl = loadTextContent('../lib/cloud-init.yaml.tmpl') | ||
| 77 | 124 | var scriptB64 = base64(loadTextContent('../lib/mitosis-node.sh')) |
| 78 | 125 | var unitB64 = base64(loadTextContent('../lib/mitosis-node.service')) |
| 79 | 126 | |
| 80 | resource nsg 'Microsoft.Network/networkSecurityGroups@2024-05-01' = { | |
| 127 | resource nsg 'Microsoft.Network/networkSecurityGroups@2024-05-01' = if (mode == 'vms') { | |
| 81 | 128 | name: 'mitosis-nodes' |
| 82 | 129 | location: location |
| 83 | 130 | properties: { |
| @@ -114,7 +161,7 @@ resource nsg 'Microsoft.Network/networkSecurityGroups@2024-05-01' = { | ||
| 114 | 161 | } |
| 115 | 162 | } |
| 116 | 163 | |
| 117 | resource vnet 'Microsoft.Network/virtualNetworks@2024-05-01' = { | |
| 164 | resource vnet 'Microsoft.Network/virtualNetworks@2024-05-01' = if (mode == 'vms') { | |
| 118 | 165 | name: 'mitosis' |
| 119 | 166 | location: location |
| 120 | 167 | properties: { |
| @@ -240,23 +287,214 @@ resource vm 'Microsoft.Compute/virtualMachines@2024-07-01' = [ | ||
| 240 | 287 | } |
| 241 | 288 | ] |
| 242 | 289 | |
| 290 | // ═════════════════════════════════ aks mode ═════════════════════════════════ | |
| 291 | // A managed ground: no cloud-init, no node contract — the operator germinates | |
| 292 | // against the kubeconfig (CLUSTER_PROFILE=aks). Everything a strict | |
| 293 | // "Azure-pure" bar demands is provisioned here so germinate only consumes IDs. | |
| 294 | ||
| 295 | resource logs 'Microsoft.OperationalInsights/workspaces@2023-09-01' = if (mode == 'aks') { | |
| 296 | name: 'mitosis-logs' | |
| 297 | location: location | |
| 298 | properties: { | |
| 299 | sku: { name: 'PerGB2018' } | |
| 300 | retentionInDays: 30 | |
| 301 | } | |
| 302 | } | |
| 303 | ||
| 304 | resource acr 'Microsoft.ContainerRegistry/registries@2023-07-01' = if (mode == 'aks') { | |
| 305 | name: acrName | |
| 306 | location: location | |
| 307 | sku: { name: 'Standard' } | |
| 308 | properties: { | |
| 309 | adminUserEnabled: false | |
| 310 | } | |
| 311 | } | |
| 312 | ||
| 313 | resource aks 'Microsoft.ContainerService/managedClusters@2024-05-01' = if (mode == 'aks') { | |
| 314 | name: 'mitosis-aks' | |
| 315 | location: location | |
| 316 | identity: { type: 'SystemAssigned' } | |
| 317 | properties: { | |
| 318 | dnsPrefix: 'mitosis' | |
| 319 | kubernetesVersion: empty(aksVersion) ? null : aksVersion | |
| 320 | agentPoolProfiles: [ | |
| 321 | { | |
| 322 | name: 'system' | |
| 323 | mode: 'System' | |
| 324 | count: aksNodeCount | |
| 325 | vmSize: aksVmSize | |
| 326 | osType: 'Linux' | |
| 327 | osDiskSizeGB: 80 | |
| 328 | availabilityZones: empty(aksZones) ? null : aksZones | |
| 329 | } | |
| 330 | ] | |
| 331 | // Workload identity: cert-manager (DNS-01) and Flux (Key Vault sops) | |
| 332 | // federate below against this issuer — no static credentials in-cluster. | |
| 333 | oidcIssuerProfile: { enabled: true } | |
| 334 | securityProfile: { workloadIdentity: { enabled: true } } | |
| 335 | networkProfile: { | |
| 336 | networkPlugin: 'azure' | |
| 337 | networkPluginMode: 'overlay' | |
| 338 | networkDataplane: 'cilium' | |
| 339 | // Kyverno GENERATES per-tenant NetworkPolicies (plat-ns-network-governance); | |
| 340 | // cilium is the overlay-compatible enforcer that actually honors them. | |
| 341 | networkPolicy: 'cilium' | |
| 342 | } | |
| 343 | // Container Insights (logs) — the Azure-pure replacement for the k3s | |
| 344 | // recipe's Grafana, together with managed Prometheus below. | |
| 345 | addonProfiles: { | |
| 346 | omsagent: { | |
| 347 | enabled: true | |
| 348 | config: { | |
| 349 | logAnalyticsWorkspaceResourceID: logs.id | |
| 350 | useAADAuth: 'true' | |
| 351 | } | |
| 352 | } | |
| 353 | } | |
| 354 | // Managed Prometheus (metrics into Azure Monitor workspace-less mode). | |
| 355 | azureMonitorProfile: { | |
| 356 | metrics: { | |
| 357 | enabled: true | |
| 358 | kubeStateMetrics: {} | |
| 359 | } | |
| 360 | } | |
| 361 | } | |
| 362 | } | |
| 363 | ||
| 364 | // kubelet pulls infra images + ci-runner from ACR — AcrPull on the kubelet identity. | |
| 365 | resource acrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (mode == 'aks') { | |
| 366 | name: guid(resourceGroup().id, 'acrpull', acrName) | |
| 367 | scope: acr | |
| 368 | properties: { | |
| 369 | principalId: aks.properties.identityProfile.kubeletidentity.objectId | |
| 370 | roleDefinitionId: subscriptionResourceId( | |
| 371 | 'Microsoft.Authorization/roleDefinitions', | |
| 372 | '7f951dda-4ed3-4680-a7ca-43fe172d538d' // AcrPull | |
| 373 | ) | |
| 374 | principalType: 'ServicePrincipal' | |
| 375 | } | |
| 376 | } | |
| 377 | ||
| 378 | // Public zone for the platform domain — delegate its NS records at the | |
| 379 | // registrar; cert-manager answers DNS-01 challenges here via workload identity. | |
| 380 | resource dnsZone 'Microsoft.Network/dnsZones@2023-07-01-preview' = if (mode == 'aks' && createDnsZone) { | |
| 381 | name: domain | |
| 382 | location: 'global' | |
| 383 | } | |
| 384 | ||
| 385 | resource certManagerId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if (mode == 'aks') { | |
| 386 | name: 'mitosis-cert-manager' | |
| 387 | location: location | |
| 388 | } | |
| 389 | ||
| 390 | resource certManagerFed 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = if (mode == 'aks') { | |
| 391 | parent: certManagerId | |
| 392 | name: 'cert-manager' | |
| 393 | properties: { | |
| 394 | issuer: aks.properties.oidcIssuerProfile.issuerURL | |
| 395 | subject: 'system:serviceaccount:cert-manager:cert-manager' | |
| 396 | audiences: ['api://AzureADTokenExchange'] | |
| 397 | } | |
| 398 | } | |
| 399 | ||
| 400 | resource certManagerDns 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (mode == 'aks' && createDnsZone) { | |
| 401 | name: guid(resourceGroup().id, 'certmgr-dns', domain) | |
| 402 | scope: dnsZone | |
| 403 | properties: { | |
| 404 | principalId: certManagerId.properties.principalId | |
| 405 | roleDefinitionId: subscriptionResourceId( | |
| 406 | 'Microsoft.Authorization/roleDefinitions', | |
| 407 | 'befefa01-2a29-4197-83a8-272ff33ce314' // DNS Zone Contributor | |
| 408 | ) | |
| 409 | principalType: 'ServicePrincipal' | |
| 410 | } | |
| 411 | } | |
| 412 | ||
| 413 | // Key Vault holds the sops KMS key at pilot hardening (sovereign custody the | |
| 414 | // Azure way). Created now so the identity plumbing ships with the ground. | |
| 415 | resource kv 'Microsoft.KeyVault/vaults@2023-07-01' = if (mode == 'aks') { | |
| 416 | name: keyVaultName | |
| 417 | location: location | |
| 418 | properties: { | |
| 419 | sku: { family: 'A', name: 'standard' } | |
| 420 | tenantId: subscription().tenantId | |
| 421 | enableRbacAuthorization: true | |
| 422 | enableSoftDelete: true | |
| 423 | } | |
| 424 | } | |
| 425 | ||
| 426 | resource fluxSopsId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if (mode == 'aks') { | |
| 427 | name: 'mitosis-flux-sops' | |
| 428 | location: location | |
| 429 | } | |
| 430 | ||
| 431 | resource fluxSopsFed 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = if (mode == 'aks') { | |
| 432 | parent: fluxSopsId | |
| 433 | name: 'kustomize-controller' | |
| 434 | properties: { | |
| 435 | issuer: aks.properties.oidcIssuerProfile.issuerURL | |
| 436 | subject: 'system:serviceaccount:flux-system:kustomize-controller' | |
| 437 | audiences: ['api://AzureADTokenExchange'] | |
| 438 | } | |
| 439 | } | |
| 440 | ||
| 441 | resource fluxSopsKv 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (mode == 'aks') { | |
| 442 | name: guid(resourceGroup().id, 'flux-sops-kv', keyVaultName) | |
| 443 | scope: kv | |
| 444 | properties: { | |
| 445 | principalId: fluxSopsId.properties.principalId | |
| 446 | roleDefinitionId: subscriptionResourceId( | |
| 447 | 'Microsoft.Authorization/roleDefinitions', | |
| 448 | '12338af0-0e69-4776-bea7-57ae8d297424' // Key Vault Crypto User | |
| 449 | ) | |
| 450 | principalType: 'ServicePrincipal' | |
| 451 | } | |
| 452 | } | |
| 453 | ||
| 454 | // ═════════════════════════════════ outputs ══════════════════════════════════ | |
| 455 | ||
| 243 | 456 | // for-expressions may not nest inside join()/interpolation, and variable |
| 244 | 457 | // loops can't touch runtime values — so the joined list is a ternary (ARM's |
| 245 | 458 | // if() only evaluates the taken branch, so pip[1]/pip[2] resolve only in HA). |
| 246 | var allIps = ha | |
| 247 | ? '${pip[0].properties.ipAddress}, ${pip[1].properties.ipAddress}, ${pip[2].properties.ipAddress}' | |
| 248 | : pip[0].properties.ipAddress | |
| 459 | var allIps = mode != 'vms' | |
| 460 | ? '' | |
| 461 | : ha | |
| 462 | ? '${pip[0].properties.ipAddress}, ${pip[1].properties.ipAddress}, ${pip[2].properties.ipAddress}' | |
| 463 | : pip[0].properties.ipAddress | |
| 464 | var leaderIp = mode == 'vms' ? pip[0].properties.ipAddress : '' | |
| 249 | 465 | |
| 466 | output mode string = mode | |
| 250 | 467 | output nodeIps array = [for i in range(0, nodeCount): pip[i].properties.ipAddress] |
| 251 | output dnsRecord string = '*.${domain} A ${allIps}' | |
| 252 | output nextSteps string = join( | |
| 253 | [ | |
| 254 | 'Germination runs ~30-45 min on the leader. From here:' | |
| 255 | ' watch it: ssh ${adminUsername}@${pip[0].properties.ipAddress} journalctl -u mitosis-node -f' | |
| 256 | ' DNS: create *.${domain} A ${allIps}' | |
| 257 | ' when done: ssh ${adminUsername}@${pip[0].properties.ipAddress} sudo cat /root/PLATFORM.md' | |
| 258 | 'The sovereign key + admin logins are on the leader — PLATFORM.md walks you through' | |
| 259 | 'copying the key somewhere safe (do it first).' | |
| 260 | ], | |
| 261 | '\n' | |
| 262 | ) | |
| 468 | output dnsRecord string = mode == 'vms' | |
| 469 | ? '*.${domain} A ${allIps}' | |
| 470 | : 'delegate NS for ${domain} to the dnsNameServers output; apps resolve via the zone' | |
| 471 | output acrLoginServer string = mode == 'aks' ? acr.properties.loginServer : '' | |
| 472 | output aksName string = mode == 'aks' ? aks.name : '' | |
| 473 | output oidcIssuer string = mode == 'aks' ? aks.properties.oidcIssuerProfile.issuerURL : '' | |
| 474 | output dnsNameServers array = mode == 'aks' && createDnsZone ? dnsZone.properties.nameServers : [] | |
| 475 | output keyVaultName string = mode == 'aks' ? kv.name : '' | |
| 476 | output certManagerClientId string = mode == 'aks' ? certManagerId.properties.clientId : '' | |
| 477 | output fluxSopsClientId string = mode == 'aks' ? fluxSopsId.properties.clientId : '' | |
| 478 | output nextSteps string = mode == 'vms' | |
| 479 | ? join( | |
| 480 | [ | |
| 481 | 'Germination runs ~30-45 min on the leader. From here:' | |
| 482 | ' watch it: ssh ${adminUsername}@${leaderIp} journalctl -u mitosis-node -f' | |
| 483 | ' DNS: create *.${domain} A ${allIps}' | |
| 484 | ' when done: ssh ${adminUsername}@${leaderIp} sudo cat /root/PLATFORM.md' | |
| 485 | 'The sovereign key + admin logins are on the leader — PLATFORM.md walks you through' | |
| 486 | 'copying the key somewhere safe (do it first).' | |
| 487 | ], | |
| 488 | '\n' | |
| 489 | ) | |
| 490 | : join( | |
| 491 | [ | |
| 492 | 'AKS ground is up. From here:' | |
| 493 | ' 1. delegate NS for ${domain} at your registrar (dnsNameServers output)' | |
| 494 | ' 2. mirror infra images: deploy/azure/mirror-images.sh <acrLoginServer>' | |
| 495 | ' 3. kubeconfig: az aks get-credentials -g <rg> -n mitosis-aks' | |
| 496 | ' 4. germinate: CLUSTER_PROFILE=aks ACR=<acrLoginServer> DOMAIN=${domain} bin/germinate' | |
| 497 | 'The sovereign key is minted on YOUR machine during germination - back it up first thing.' | |
| 498 | ], | |
| 499 | '\n' | |
| 500 | ) |
deploy/azure/mirror-images.sh added +70
| @@ -0,0 +1,70 @@ | ||
| 1 | #!/usr/bin/env bash | |
| 2 | # mirror-images.sh — copy the platform's external infra images into ACR. | |
| 3 | # | |
| 4 | # deploy/azure/mirror-images.sh <acr-name-or-login-server> | |
| 5 | # | |
| 6 | # Part of the AKS ("Azure-pure") ground: nodes pull ONLY from ACR and from the | |
| 7 | # platform's own registry (git.<domain>). This imports every third-party image | |
| 8 | # the recipe pins so no pod ever reaches docker.io/ghcr/quay at runtime. | |
| 9 | # `az acr import` copies server-side — nothing is pulled through this machine. | |
| 10 | # | |
| 11 | # Version pins mirror clusters/local/platform/*/helmrelease.yaml at the vintage | |
| 12 | # this script last shipped; germinate's aks profile rewrites the HelmRelease | |
| 13 | # values to $ACR/<same path>. Keep the two in sync when bumping charts. | |
| 14 | # k3s-only images (rancher/klipper-lb, local-path-provisioner, mirrored-*) are | |
| 15 | # deliberately absent — they have no AKS counterpart. | |
| 16 | set -euo pipefail | |
| 17 | ||
| 18 | ACR="" | |
| 19 | ACR="" # accept either form | |
| 20 | ||
| 21 | IMAGES=( | |
| 22 | # docker.io — core infra | |
| 23 | "docker.io/library/traefik:v3.5" | |
| 24 | "docker.io/library/alpine:3.20" | |
| 25 | "docker.io/library/caddy:2-alpine" | |
| 26 | "docker.io/alpine/git:2.43.0" | |
| 27 | "quay.io/minio/minio:RELEASE.2025-02-07T23-21-09Z" # minio left docker.io; quay is canonical | |
| 28 | "quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z" | |
| 29 | # flux (installed by `flux install` — mirror for restrict-image-registries compliance) | |
| 30 | "ghcr.io/fluxcd/helm-controller:v1.5.5" | |
| 31 | "ghcr.io/fluxcd/kustomize-controller:v1.8.5" | |
| 32 | "ghcr.io/fluxcd/source-controller:v1.8.5" | |
| 33 | "ghcr.io/fluxcd/notification-controller:v1.8.4" | |
| 34 | # keda | |
| 35 | "ghcr.io/kedacore/keda:2.20.0" | |
| 36 | "ghcr.io/kedacore/keda-metrics-apiserver:2.20.0" | |
| 37 | "ghcr.io/kedacore/keda-admission-webhooks:2.20.0" | |
| 38 | # cnpg | |
| 39 | "ghcr.io/cloudnative-pg/cloudnative-pg:1.30.0" | |
| 40 | "ghcr.io/cloudnative-pg/postgresql:18-system-trixie" | |
| 41 | # app-template tenant DBs pin the patch tag (kyverno aks-image-redirect rewrites them to ACR) | |
| 42 | "ghcr.io/cloudnative-pg/postgresql:18.4-system-trixie" | |
| 43 | # sablier | |
| 44 | "ghcr.io/sablierapp/sablier:1.16.1" | |
| 45 | # cert-manager | |
| 46 | "quay.io/jetstack/cert-manager-controller:v1.21.1" | |
| 47 | "quay.io/jetstack/cert-manager-cainjector:v1.21.1" | |
| 48 | "quay.io/jetstack/cert-manager-webhook:v1.21.1" | |
| 49 | # kyverno | |
| 50 | "reg.kyverno.io/kyverno/kyverno:v1.18.2" | |
| 51 | "reg.kyverno.io/kyverno/kyverno-cli:v1.18.2" | |
| 52 | "reg.kyverno.io/kyverno/background-controller:v1.18.2" | |
| 53 | "reg.kyverno.io/kyverno/reports-controller:v1.18.2" | |
| 54 | # forgejo (the product's own VCS+registry, pulled as an image at install) | |
| 55 | "code.forgejo.org/forgejo/forgejo:15.0.5-rootless" | |
| 56 | ) | |
| 57 | ||
| 58 | fail=0 | |
| 59 | for src in ""; do | |
| 60 | # keep the source path under ACR verbatim (minus registry host) so germinate | |
| 61 | # can rewrite registries by prefix only: ghcr.io/X:Y -> $ACR/X:Y | |
| 62 | path="" | |
| 63 | echo "==> -> /" | |
| 64 | if ! az acr import --name "" --source "" --image "" --force >/dev/null 2>&1; then | |
| 65 | echo " RETRY with explicit registry auth-free import" | |
| 66 | az acr import --name "" --source "" --image "" --force || { echo " FAILED: "; fail=1; } | |
| 67 | fi | |
| 68 | done | |
| 69 | ||
| 70 | [ "" = 0 ] && echo "==> all images mirrored into .azurecr.io" || { echo "==> SOME IMPORTS FAILED — rerun; nothing partial is harmful"; exit 1; } |
deploy/azure/servers.bicep added +193
| @@ -0,0 +1,193 @@ | ||
| 1 | // Mitosis on Azure — in-place HA expansion: add server nodes to a LIVE platform. | |
| 2 | // | |
| 3 | // az deployment group create -g mitosis -f servers.bicep \ | |
| 4 | // -p domain=plat.example.com -p adminSshPublicKey="$(cat ~/.ssh/id_ed25519.pub)" \ | |
| 5 | // -p k3sToken="$(ssh <leader> sudo cat /var/lib/rancher/k3s/server/token)" | |
| 6 | // | |
| 7 | // main.bicep's ha=true is for FRESH deployments only: re-running it against a | |
| 8 | // live group would trip on the leader VM's immutable customData and REPLACE the | |
| 9 | // NSG rule set (deleting any operator-added rules). This template creates ONLY | |
| 10 | // the joiner nodes (mitosis-server-1/2, 10.60.0.11/.12), referencing the | |
| 11 | // existing vnet/subnet and touching nothing else. | |
| 12 | // | |
| 13 | // Prerequisite: the leader must already run embedded etcd. On a sqlite | |
| 14 | // single-node install, write `cluster-init: true` to /etc/rancher/k3s/config.yaml | |
| 15 | // and restart k3s (k3s migrates sqlite -> etcd in place); THEN deploy this. | |
| 16 | // | |
| 17 | // The node contract (env file + bootstrap + unit + registry recipe) is the same | |
| 18 | // deploy/lib + recipe/ template main.bicep renders — joiners run ROLE=server: | |
| 19 | // join the leader, build their own ci-runner, and never germinate. | |
| 20 | ||
| 21 | @description('The platform\'s sovereign domain, e.g. plat.example.com.') | |
| 22 | param domain string | |
| 23 | ||
| 24 | @description('Your SSH public key (contents of ~/.ssh/id_ed25519.pub).') | |
| 25 | param adminSshPublicKey string | |
| 26 | ||
| 27 | @description('The LIVE cluster\'s join token — read it from the leader: sudo cat /var/lib/rancher/k3s/server/token. (Not a fresh GUID: joiners must present the existing cluster\'s token.)') | |
| 28 | @secure() | |
| 29 | param k3sToken string | |
| 30 | ||
| 31 | @description('VM login user (must match the leader\'s).') | |
| 32 | param adminUsername string = 'mitosis' | |
| 33 | ||
| 34 | @description('Match the leader\'s size unless capacity-planning says otherwise.') | |
| 35 | param vmSize string = 'Standard_D4as_v7' | |
| 36 | ||
| 37 | @description('OS disk size, GB.') | |
| 38 | param diskGb int = 80 | |
| 39 | ||
| 40 | @description('k3s release — MUST match the version the leader runs (k3s --version).') | |
| 41 | param k3sVersion string = 'v1.31.5+k3s1' | |
| 42 | ||
| 43 | @description('Where joiners clone mitosis from (ci-runner bootstrap). Point at the live platform\'s own mirror for sovereignty.') | |
| 44 | param mitosisUrl string = 'https://git.open-platform.sh/plat/mitosis' | |
| 45 | ||
| 46 | @description('Branch or tag of mitosis.') | |
| 47 | param mitosisRef string = 'main' | |
| 48 | ||
| 49 | @description('Availability zone per joiner (index 0 -> mitosis-server-1, 1 -> mitosis-server-2). Empty string = non-zonal, matching a non-zonal leader.') | |
| 50 | param zones array = ['2', '3'] | |
| 51 | ||
| 52 | @description('The leader\'s static private IP on the nodes subnet.') | |
| 53 | param leaderIp string = '10.60.0.10' | |
| 54 | ||
| 55 | param location string = resourceGroup().location | |
| 56 | ||
| 57 | // Joiners are indexed 1..2 — .11/.12 next to the leader's .10 (main.bicep contract). | |
| 58 | var serverIndexes = [1, 2] | |
| 59 | ||
| 60 | resource vnet 'Microsoft.Network/virtualNetworks@2024-05-01' existing = { | |
| 61 | name: 'mitosis' | |
| 62 | } | |
| 63 | ||
| 64 | resource subnet 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' existing = { | |
| 65 | parent: vnet | |
| 66 | name: 'nodes' | |
| 67 | } | |
| 68 | ||
| 69 | var nodeEnv = [ | |
| 70 | for i in serverIndexes: join( | |
| 71 | [ | |
| 72 | '# /etc/mitosis/node.env — written at provision time; read by mitosis-node.sh' | |
| 73 | 'CLOUD=\'azure\'' | |
| 74 | 'ROLE=\'server\'' | |
| 75 | 'HA=\'true\'' | |
| 76 | 'DOMAIN=\'${domain}\'' | |
| 77 | 'LEADER_IP=\'${leaderIp}\'' | |
| 78 | 'K3S_TOKEN=\'${k3sToken}\'' | |
| 79 | 'K3S_VERSION=\'${k3sVersion}\'' | |
| 80 | 'MITOSIS_URL=\'${mitosisUrl}\'' | |
| 81 | 'MITOSIS_REF=\'${mitosisRef}\'' | |
| 82 | 'SSH_USER=\'${adminUsername}\'' | |
| 83 | 'CLAUDE_CODE_OAUTH_TOKEN=\'\'' | |
| 84 | 'ANTHROPIC_API_KEY=\'\'' | |
| 85 | ], | |
| 86 | '\n' | |
| 87 | ) | |
| 88 | ] | |
| 89 | ||
| 90 | var registries = replace( | |
| 91 | loadTextContent('../../recipe/registries.yaml.tmpl'), | |
| 92 | '__DOMAIN__', | |
| 93 | domain | |
| 94 | ) | |
| 95 | var cloudInitTmpl = loadTextContent('../lib/cloud-init.yaml.tmpl') | |
| 96 | var cloudInit = [ | |
| 97 | for (idx, j) in serverIndexes: replace( | |
| 98 | replace( | |
| 99 | replace( | |
| 100 | replace(cloudInitTmpl, '\${registries_b64}', base64(registries)), | |
| 101 | '\${node_env_b64}', | |
| 102 | base64(nodeEnv[j]) | |
| 103 | ), | |
| 104 | '\${script_b64}', | |
| 105 | base64(loadTextContent('../lib/mitosis-node.sh')) | |
| 106 | ), | |
| 107 | '\${unit_b64}', | |
| 108 | base64(loadTextContent('../lib/mitosis-node.service')) | |
| 109 | ) | |
| 110 | ] | |
| 111 | ||
| 112 | resource pip 'Microsoft.Network/publicIPAddresses@2024-05-01' = [ | |
| 113 | for i in serverIndexes: { | |
| 114 | name: 'mitosis-server-${i}' | |
| 115 | location: location | |
| 116 | sku: { name: 'Standard' } | |
| 117 | properties: { publicIPAllocationMethod: 'Static' } | |
| 118 | } | |
| 119 | ] | |
| 120 | ||
| 121 | resource nic 'Microsoft.Network/networkInterfaces@2024-05-01' = [ | |
| 122 | for (idx, j) in serverIndexes: { | |
| 123 | name: 'mitosis-server-${idx}' | |
| 124 | location: location | |
| 125 | properties: { | |
| 126 | ipConfigurations: [ | |
| 127 | { | |
| 128 | name: 'primary' | |
| 129 | properties: { | |
| 130 | subnet: { id: subnet.id } | |
| 131 | privateIPAllocationMethod: 'Static' | |
| 132 | privateIPAddress: '10.60.0.${10 + idx}' | |
| 133 | publicIPAddress: { id: pip[j].id } | |
| 134 | } | |
| 135 | } | |
| 136 | ] | |
| 137 | } | |
| 138 | } | |
| 139 | ] | |
| 140 | ||
| 141 | resource vm 'Microsoft.Compute/virtualMachines@2024-07-01' = [ | |
| 142 | for (idx, j) in serverIndexes: { | |
| 143 | name: 'mitosis-server-${idx}' | |
| 144 | location: location | |
| 145 | zones: empty(zones[j]) ? null : [zones[j]] | |
| 146 | properties: { | |
| 147 | hardwareProfile: { vmSize: vmSize } | |
| 148 | storageProfile: { | |
| 149 | imageReference: { | |
| 150 | publisher: 'Canonical' | |
| 151 | offer: 'ubuntu-24_04-lts' | |
| 152 | sku: 'server' | |
| 153 | version: 'latest' | |
| 154 | } | |
| 155 | osDisk: { | |
| 156 | createOption: 'FromImage' | |
| 157 | diskSizeGB: diskGb | |
| 158 | managedDisk: { storageAccountType: 'StandardSSD_LRS' } | |
| 159 | } | |
| 160 | } | |
| 161 | osProfile: { | |
| 162 | computerName: 'mitosis-server-${idx}' | |
| 163 | adminUsername: adminUsername | |
| 164 | customData: base64(cloudInit[j]) | |
| 165 | linuxConfiguration: { | |
| 166 | disablePasswordAuthentication: true | |
| 167 | ssh: { | |
| 168 | publicKeys: [ | |
| 169 | { | |
| 170 | path: '/home/${adminUsername}/.ssh/authorized_keys' | |
| 171 | keyData: adminSshPublicKey | |
| 172 | } | |
| 173 | ] | |
| 174 | } | |
| 175 | } | |
| 176 | } | |
| 177 | networkProfile: { | |
| 178 | networkInterfaces: [{ id: nic[j].id }] | |
| 179 | } | |
| 180 | } | |
| 181 | } | |
| 182 | ] | |
| 183 | ||
| 184 | output serverIps array = [for (idx, j) in serverIndexes: pip[j].properties.ipAddress] | |
| 185 | output nextSteps string = join( | |
| 186 | [ | |
| 187 | 'Joiners boot, wait for the leader apiserver, and join etcd (~5-10 min).' | |
| 188 | ' watch: ssh ${adminUsername}@<serverIp> journalctl -u mitosis-node -f' | |
| 189 | ' verify: kubectl get nodes (3 Ready, roles control-plane,etcd,master)' | |
| 190 | ' DNS: add each server IP to the *.${domain} record set (round-robin edge).' | |
| 191 | ], | |
| 192 | '\n' | |
| 193 | ) |
new above
offline